Now supports entire Python 2.0 language and still supports Python

1.5.2.  The compiler generates code for the version of the interpreter
it is run under.

ast.py:
    Print and Printnl add dest attr for extended print
    new node AugAssign for augmented assignments
    new nodes ListComp, ListCompFor, and ListCompIf for list
        comprehensions

pyassem.py:
    add work around for string-Unicode comparison raising UnicodeError
        on comparison of two objects in code object's const table

pycodegen.py:
    define VERSION, the Python major version number
    get magic number using imp.get_magic() instead of hard coding
    implement list comprehensions, extended print, and augmented
        assignment; augmented assignment uses Delegator classes (see
        doc string)
    fix import and tuple unpacking for 1.5.2

transformer.py:
    various changes to support new 2.0 grammar and old 1.5 grammar
    add debug_tree helper than converts and symbol and token numbers
    to their names

Lib/compiler/ast.py

@@ -279,22 +279,24 @@ class Const(Node):
 class Print(Node):
   nodes['print'] = 'Print'
 
-  def __init__(self, nodes):
+  def __init__(self, nodes, dest):
     self.nodes = nodes
-    self._children = ('print', nodes)
+    self.dest = dest
+    self._children = ('print', nodes, dest)
 
   def __repr__(self):
-    return "Print(%s)" % self._children[1:]
+    return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
 
 class Printnl(Node):
   nodes['printnl'] = 'Printnl'
 
-  def __init__(self, nodes):
+  def __init__(self, nodes, dest):
     self.nodes = nodes
-    self._children = ('printnl', nodes)
+    self.dest = dest
+    self._children = ('printnl', nodes, dest)
 
   def __repr__(self):
-    return "Printnl(%s)" % self._children[1:]
+    return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
 
 class Discard(Node):
   nodes['discard'] = 'Discard'
@@ -306,6 +308,18 @@ class Discard(Node):
   def __repr__(self):
     return "Discard(%s)" % self._children[1:]
 
+class AugAssign(Node):
+  nodes['augassign'] = 'AugAssign'
+
+  def __init__(self, node, op, expr):
+    self.node = node
+    self.op = op
+    self.expr = expr
+    self._children = ('augassign', node, op, expr)
+
+  def __repr__(self):
+    return "AugAssign(%s)" % str(self._children[1:])
+
 class Assign(Node):
   nodes['assign'] = 'Assign'
 
@@ -360,6 +374,41 @@ class AssAttr(Node):
   def __repr__(self):
     return "AssAttr(%s,%s,%s)" % self._children[1:]
 
+class ListComp(Node):
+  nodes['listcomp'] = 'ListComp'
+
+  def __init__(self, expr, quals):
+    self.expr = expr
+    self.quals = quals
+    self._children = ('listcomp', expr, quals)
+
+  def __repr__(self):
+    return "ListComp(%s, %s)" % self._children[1:]
+
+class ListCompFor(Node):
+  nodes['listcomp_for'] = 'ListCompFor'
+
+  # transformer fills in ifs after node is created
+
+  def __init__(self, assign, list, ifs):
+    self.assign = assign
+    self.list = list
+    self.ifs = ifs
+    self._children = ('listcomp_for', assign, list, ifs)
+
+  def __repr__(self):
+    return "ListCompFor(%s, %s, %s)" % self._children[1:]
+
+class ListCompIf(Node):
+  nodes['listcomp_if'] = 'ListCompIf'
+
+  def __init__(self, test):
+    self.test = test
+    self._children = ('listcomp_if', test)
+
+  def __repr__(self):
+    return "ListCompIf(%s)" % self._children[1:]
+
 class List(Node):
   nodes['list'] = 'List'
 

Lib/compiler/pyassem.py

@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
 
     def _lookupName(self, name, list):
         """Return index of name in list, appending if necessary"""
-        if name in list:
+        found = None
-            i = list.index(name)
+        t = type(name)
+        for i in range(len(list)):
+            # must do a comparison on type first to prevent UnicodeErrors 
+            if t == type(list[i]) and list[i] == name:
+                found = 1
+                break
+        if found:
             # this is cheap, but incorrect in some cases, e.g 2 vs. 2L
             if type(name) == type(list[i]):
                 return i

Lib/compiler/pycodegen.py

@@ -1,8 +1,10 @@
+import imp
 import os
 import marshal
 import stat
 import string
 import struct
+import sys
 import types
 from cStringIO import StringIO
 
@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
 from compiler import pyassem, misc
 from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
 
+# Do we have Python 1.x or Python 2.x?
+try:
+    VERSION = sys.version_info[0]
+except AttributeError:
+    VERSION = 1
+
 callfunc_opcode_info = {
     # (Have *args, Have **args) : opcode
     (0,0) : "CALL_FUNCTION",
@@ -18,12 +26,12 @@ callfunc_opcode_info = {
     (1,1) : "CALL_FUNCTION_VAR_KW",
 }
 
-def compile(filename):
+def compile(filename, display=0):
     f = open(filename)
     buf = f.read()
     f.close()
     mod = Module(buf, filename)
-    mod.compile()
+    mod.compile(display)
     f = open(filename + "c", "wb")
     mod.dump(f)
     f.close()
@@ -34,28 +42,30 @@ class Module:
         self.source = source
         self.code = None
 
-    def compile(self):
+    def compile(self, display=0):
         ast = parse(self.source)
         root, filename = os.path.split(self.filename)
         gen = ModuleCodeGenerator(filename)
         walk(ast, gen, 1)
+        if display:
+            import pprint
+            print pprint.pprint(ast)
         self.code = gen.getCode()
 
     def dump(self, f):
         f.write(self.getPycHeader())
         marshal.dump(self.code, f)
 
-    MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
+    MAGIC = imp.get_magic()
 
     def getPycHeader(self):
         # compile.c uses marshal to write a long directly, with
         # calling the interface that would also generate a 1-byte code
         # to indicate the type of the value.  simplest way to get the
         # same effect is to call marshal and then skip the code.
-        magic = marshal.dumps(self.MAGIC)[1:]
         mtime = os.stat(self.filename)[stat.ST_MTIME]
         mtime = struct.pack('i', mtime)
-        return magic + mtime
+        return self.MAGIC + mtime
 
 class CodeGenerator:
 
@@ -63,7 +73,7 @@ class CodeGenerator:
 
     def __init__(self, filename):
 ## Subclasses must define a constructor that intializes self.graph
-## before calling this init function
+## before calling this init function, e.g.
 ##         self.graph = pyassem.PyFlowGraph()
         self.filename = filename
         self.locals = misc.Stack()
@@ -142,7 +152,6 @@ class CodeGenerator:
 
     def visitLambda(self, node):
         self._visitFuncOrLambda(node, isLambda=1)
-##        self.storeName("<lambda>")
 
     def _visitFuncOrLambda(self, node, isLambda):
         gen = FunctionCodeGenerator(node, self.filename, isLambda)
@@ -180,10 +189,6 @@ class CodeGenerator:
             test, suite = node.tests[i]
             self.set_lineno(test)
             self.visit(test)
-##            if i == numtests - 1 and not node.else_:
-##                nextTest = end
-##            else:
-##                nextTest = self.newBlock()
             nextTest = self.newBlock()
             self.emit('JUMP_IF_FALSE', nextTest)
             self.nextBlock()
@@ -304,6 +309,70 @@ class CodeGenerator:
             self.emit('POP_TOP')
             self.nextBlock(end)
 
+    # list comprehensions
+    __list_count = 0
+    
+    def visitListComp(self, node):
+        # XXX would it be easier to transform the AST into the form it
+        # would have if the list comp were expressed as a series of
+        # for and if stmts and an explicit append?
+        self.set_lineno(node)
+        # setup list
+        append = "$append%d" % self.__list_count
+        self.__list_count = self.__list_count + 1
+        self.emit('BUILD_LIST', 0)
+        self.emit('DUP_TOP')
+        self.emit('LOAD_ATTR', 'append')
+        self.storeName(append)
+        l = len(node.quals)
+        stack = []
+        for i, for_ in zip(range(l), node.quals):
+            start, anchor = self.visit(for_)
+            cont = None
+            for if_ in for_.ifs:
+                if cont is None:
+                    cont = self.newBlock()
+                self.visit(if_, cont)
+            stack.insert(0, (start, cont, anchor))
+            
+        self.loadName(append)
+        self.visit(node.expr)
+        self.emit('CALL_FUNCTION', 1)
+        self.emit('POP_TOP')
+        
+        for start, cont, anchor in stack:
+            if cont:
+                skip_one = self.newBlock()
+                self.emit('JUMP_FORWARD', skip_one)
+                self.nextBlock(cont)
+                self.emit('POP_TOP')
+                self.nextBlock(skip_one)
+            self.emit('JUMP_ABSOLUTE', start)
+            self.nextBlock(anchor)
+        self.delName(append)
+        
+        self.__list_count = self.__list_count - 1
+
+    def visitListCompFor(self, node):
+        self.set_lineno(node)
+        start = self.newBlock()
+        anchor = self.newBlock()
+
+        self.visit(node.list)
+        self.visit(ast.Const(0))
+        self.emit('SET_LINENO', node.lineno)
+        self.nextBlock(start)
+        self.emit('FOR_LOOP', anchor)
+        self.visit(node.assign)
+        return start, anchor
+
+    def visitListCompIf(self, node, branch):
+        self.set_lineno(node)
+        self.visit(node.test)
+        self.emit('JUMP_IF_FALSE', branch)
+        self.newBlock()
+        self.emit('POP_TOP')
+
     # exception related
 
     def visitAssert(self, node):
@@ -397,10 +466,6 @@ class CodeGenerator:
 
     # misc
 
-##     def visitStmt(self, node):
-##         # nothing to do except walk the children
-##         pass
-
     def visitDiscard(self, node):
         self.visit(node.expr)
         self.emit('POP_TOP')
@@ -426,17 +491,20 @@ class CodeGenerator:
     def visitImport(self, node):
         self.set_lineno(node)
         for name, alias in node.names:
+            if VERSION > 1:
                 self.emit('LOAD_CONST', None)
             self.emit('IMPORT_NAME', name)
-            self._resolveDots(name)
+            mod = string.split(name, ".")[0]
-            self.storeName(alias or name)
+            self.storeName(alias or mod)
 
     def visitFrom(self, node):
         self.set_lineno(node)
         fromlist = map(lambda (name, alias): name, node.names)
+        if VERSION > 1:
             self.emit('LOAD_CONST', tuple(fromlist))
         self.emit('IMPORT_NAME', node.modname)
         for name, alias in node.names:
+            if VERSION > 1:
                 if name == '*':
                     self.namespace = 0
                     self.emit('IMPORT_STAR')
@@ -447,6 +515,8 @@ class CodeGenerator:
                     self.emit('IMPORT_FROM', name)
                     self._resolveDots(name)
                     self.storeName(alias or name)
+            else:
+                self.emit('IMPORT_FROM', name)
         self.emit('POP_TOP')
 
     def _resolveDots(self, name):
@@ -491,13 +561,85 @@ class CodeGenerator:
             print "warning: unexpected flags:", node.flags
             print node
 
-    def visitAssTuple(self, node):
+    def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
         if findOp(node) != 'OP_DELETE':
-            self.emit('UNPACK_SEQUENCE', len(node.nodes))
+            self.emit(op, len(node.nodes))
         for child in node.nodes:
             self.visit(child)
 
-    visitAssList = visitAssTuple
+    if VERSION > 1:
+        visitAssTuple = _visitAssSequence
+        visitAssList = _visitAssSequence
+    else:
+        def visitAssTuple(self, node):
+            self._visitAssSequence(node, 'UNPACK_TUPLE')
+
+        def visitAssList(self, node):
+            self._visitAssSequence(node, 'UNPACK_LIST')
+
+    # augmented assignment
+
+    def visitAugAssign(self, node):
+        aug_node = wrap_aug(node.node)
+        self.visit(aug_node, "load")
+        self.visit(node.expr)
+        self.emit(self._augmented_opcode[node.op])
+        self.visit(aug_node, "store")
+
+    _augmented_opcode = {
+        '+=' : 'INPLACE_ADD',
+        '-=' : 'INPLACE_SUBTRACT',
+        '*=' : 'INPLACE_MULTIPLY',
+        '/=' : 'INPLACE_DIVIDE',
+        '%=' : 'INPLACE_MODULO',
+        '**=': 'INPLACE_POWER',
+        '>>=': 'INPLACE_RSHIFT',
+        '<<=': 'INPLACE_LSHIFT',
+        '&=' : 'INPLACE_AND',
+        '^=' : 'INPLACE_XOR',
+        '|=' : 'INPLACE_OR',
+        }
+
+    def visitAugName(self, node, mode):
+        if mode == "load":
+            self.loadName(node.name)
+        elif mode == "store":
+            self.storeName(node.name)
+
+    def visitAugGetattr(self, node, mode):
+        if mode == "load":
+            self.visit(node.expr)
+            self.emit('DUP_TOP')
+            self.emit('LOAD_ATTR', node.attrname)
+        elif mode == "store":
+            self.emit('ROT_TWO')
+            self.emit('STORE_ATTR', node.attrname)
+
+    def visitAugSlice(self, node, mode):
+        if mode == "load":
+            self.visitSlice(node, 1)
+        elif mode == "store":
+            slice = 0
+            if node.lower:
+                slice = slice | 1
+            if node.upper:
+                slice = slice | 2
+            if slice == 0:
+                self.emit('ROT_TWO')
+            elif slice == 3:
+                self.emit('ROT_FOUR')
+            else:
+                self.emit('ROT_THREE')
+            self.emit('STORE_SLICE+%d' % slice)
+
+    def visitAugSubscript(self, node, mode):
+        if len(node.subs) > 1:
+            raise SyntaxError, "augmented assignment to tuple is not possible"
+        if mode == "load":
+            self.visitSubscript(node, 1)
+        elif mode == "store":
+            self.emit('ROT_THREE')
+            self.emit('STORE_SUBSCR')
 
     def visitExec(self, node):
         self.visit(node.expr)
@@ -533,12 +675,23 @@ class CodeGenerator:
 
     def visitPrint(self, node):
         self.set_lineno(node)
+        if node.dest:
+            self.visit(node.dest)
         for child in node.nodes:
+            if node.dest:
+                self.emit('DUP_TOP')
             self.visit(child)
+            if node.dest:
+                self.emit('ROT_TWO')
+                self.emit('PRINT_ITEM_TO')
+            else:
                 self.emit('PRINT_ITEM')
 
     def visitPrintnl(self, node):
         self.visitPrint(node)
+        if node.dest:
+            self.emit('PRINT_NEWLINE_TO')
+        else:
             self.emit('PRINT_NEWLINE')
 
     def visitReturn(self, node):
@@ -548,7 +701,8 @@ class CodeGenerator:
 
     # slice and subscript stuff
 
-    def visitSlice(self, node):
+    def visitSlice(self, node, aug_flag=None):
+        # aug_flag is used by visitAugSlice
         self.visit(node.expr)
         slice = 0
         if node.lower:
@@ -557,6 +711,13 @@ class CodeGenerator:
         if node.upper:
             self.visit(node.upper)
             slice = slice | 2
+        if aug_flag:
+            if slice == 0:
+                self.emit('DUP_TOP')
+            elif slice == 3:
+                self.emit('DUP_TOPX', 3)
+            else:
+                self.emit('DUP_TOPX', 2)
         if node.flags == 'OP_APPLY':
             self.emit('SLICE+%d' % slice)
         elif node.flags == 'OP_ASSIGN':
@@ -567,10 +728,12 @@ class CodeGenerator:
             print "weird slice", node.flags
             raise
 
-    def visitSubscript(self, node):
+    def visitSubscript(self, node, aug_flag=None):
         self.visit(node.expr)
         for sub in node.subs:
             self.visit(sub)
+        if aug_flag:
+            self.emit('DUP_TOPX', 2)
         if len(node.subs) > 1:
             self.emit('BUILD_TUPLE', len(node.subs))
         if node.flags == 'OP_APPLY':
@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
                 self.unpackSequence(arg)
                         
     def unpackSequence(self, tup):
+        if VERSION > 1:
             self.emit('UNPACK_SEQUENCE', len(tup))
+        else:
+            self.emit('UNPACK_TUPLE', len(tup))
         for elt in tup:
             if type(elt) == types.TupleType:
                 self.unpackSequence(elt)
@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
         self.emit('LOAD_LOCALS')
         self.emit('RETURN_VALUE')
 
-
 def generateArgList(arglist):
     """Generate an arg list marking TupleArgs"""
     args = []
@@ -838,6 +1003,45 @@ class OpFinder:
         elif self.op != node.flags:
             raise ValueError, "mixed ops in stmt"
 
+class Delegator:
+    """Base class to support delegation for augmented assignment nodes
+
+    To generator code for augmented assignments, we use the following
+    wrapper classes.  In visitAugAssign, the left-hand expression node
+    is visited twice.  The first time the visit uses the normal method
+    for that node .  The second time the visit uses a different method
+    that generates the appropriate code to perform the assignment.
+    These delegator classes wrap the original AST nodes in order to
+    support the variant visit methods.
+    """
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __getattr__(self, attr):
+        return getattr(self.obj, attr)
+
+class AugGetattr(Delegator):
+    pass
+
+class AugName(Delegator):
+    pass
+
+class AugSlice(Delegator):
+    pass
+
+class AugSubscript(Delegator):
+    pass
+
+wrapper = {
+    ast.Getattr: AugGetattr,
+    ast.Name: AugName,
+    ast.Slice: AugSlice,
+    ast.Subscript: AugSubscript,
+    }
+
+def wrap_aug(node):
+    return wrapper[node.__class__](node)
+
 if __name__ == "__main__":
     import sys
 

Lib/compiler/transformer.py

@@ -1,19 +1,3 @@
-#
-# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
-#
-# This module is provided under a BSD-ish license. See
-#   http://www.opensource.org/licenses/bsd-license.html
-# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
-#
-#
-# Written by Greg Stein (gstein@lyra.org)
-#        and Bill Tutt (rassilon@lima.mudlib.org)
-# February 1997.
-#
-# Support for ast.Node subclasses written and other revisions by
-#  Jeremy Hylton (jeremy@beopen.com)
-#
-
 """Parse tree transformation module.
 
 Transforms Python source code into an abstract syntax tree (AST)
@@ -24,7 +8,21 @@ parse(buf) -> AST
 parseFile(path) -> AST
 """
 
+# Original version written by Greg Stein (gstein@lyra.org)
+#                         and Bill Tutt (rassilon@lima.mudlib.org)
+# February 1997.
 #
+# Modifications and improvements for Python 2.0 by Jeremy Hylton and
+# Mark Hammond
+
+# Portions of this file are:
+# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
+#
+# This module is provided under a BSD-ish license. See
+#   http://www.opensource.org/licenses/bsd-license.html
+# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+
+
 # The output tree has the following nodes:
 #
 # Source Python line #'s appear at the end of each of all of these nodes
@@ -49,9 +47,10 @@ parseFile(path) -> AST
 # tryexcept:  trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
 # return:     valueNode
 # const:      value
-# print:      [ node1, ..., nodeN ]
+# print:      [ node1, ..., nodeN ] [, dest]
-# printnl:    [ node1, ..., nodeN ]
+# printnl:    [ node1, ..., nodeN ] [, dest]
 # discard:    exprNode
+# augassign:  node, op, expr
 # assign:     [ node1, ..., nodeN ], exprNode
 # ass_tuple:  [ node1, ..., nodeN ]
 # ass_list:   [ node1, ..., nodeN ]
@@ -97,12 +96,12 @@ parseFile(path) -> AST
 
 import ast
 import parser
+# Care must be taken to use only symbols and tokens defined in Python
+# 1.5.2 for code branches executed in 1.5.2
 import symbol
 import token
 import string
 
-import pprint
-
 error = 'walker.error'
 
 from consts import CO_VARARGS, CO_VARKEYWORDS
@@ -328,27 +327,44 @@ class Transformer:
     #
 
     def expr_stmt(self, nodelist):
-        # testlist ('=' testlist)*
+        # augassign testlist | testlist ('=' testlist)*
         exprNode = self.com_node(nodelist[-1])
         if len(nodelist) == 1:
             return Node('discard', exprNode)
+        if nodelist[1][0] == token.EQUAL:
             nodes = [ ]
             for i in range(0, len(nodelist) - 2, 2):
                 nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
             n = Node('assign', nodes, exprNode)
             n.lineno = nodelist[1][2]
+        else:
+            lval = self.com_augassign(nodelist[0])
+            op = self.com_augassign_op(nodelist[1])
+            n = Node('augassign', lval, op[1], exprNode)
+            n.lineno = op[2]
         return n
 
     def print_stmt(self, nodelist):
-        # print: (test ',')* [test]
+        # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
         items = [ ]
-        for i in range(1, len(nodelist), 2):
+        if len(nodelist) == 1:
+            start = 1
+            dest = None
+        elif nodelist[1][0] == token.RIGHTSHIFT:
+            assert len(nodelist) == 3 \
+                   or nodelist[3][0] == token.COMMA
+            dest = self.com_node(nodelist[2])
+            start = 4
+        else:
+            dest = None
+            start = 1
+        for i in range(start, len(nodelist), 2):
             items.append(self.com_node(nodelist[i]))
         if nodelist[-1][0] == token.COMMA:
-            n = Node('print', items)
+            n = Node('print', items, dest)
             n.lineno = nodelist[0][2]
             return n
-        n = Node('printnl', items)
+        n = Node('printnl', items, dest)
         n.lineno = nodelist[0][2]
         return n
 
@@ -405,15 +421,22 @@ class Transformer:
         # import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
         # from: 'from' dotted_name 'import'
         #                        ('*' | import_as_name (',' import_as_name)*)
-        names = []
-        is_as = 0
         if nodelist[0][1] == 'from':
+            names = []
+            if nodelist[3][0] == token.NAME:
+                for i in range(3, len(nodelist), 2):
+                    names.append((nodelist[i][1], None))
+            else:
                 for i in range(3, len(nodelist), 2):
                     names.append(self.com_import_as_name(nodelist[i][1]))
             n = Node('from', self.com_dotted_name(nodelist[1]), names)
             n.lineno = nodelist[0][2]
             return n
 
+        if nodelist[1][0] == symbol.dotted_name:
+            names = [(self.com_dotted_name(nodelist[1][1:]), None)]
+        else:
+            names = []
             for i in range(1, len(nodelist), 2):
                 names.append(self.com_dotted_as_name(nodelist[i]))
         n = Node('import', names)
@@ -737,7 +760,7 @@ class Transformer:
             return Node('discard', Node('const', None))
 
         if node[0] not in _legal_node_types:
-            raise error, 'illegal node passed to com_node: %s' % node[0]
+            raise error, 'illegal node passed to com_node: %s' % `node`
 
 #    print "dispatch", self._dispatch[node[0]].__name__, node
         return self._dispatch[node[0]](node[1:])
@@ -818,8 +841,11 @@ class Transformer:
 
     def com_dotted_as_name(self, node):
         dot = self.com_dotted_name(node[1])
-        if len(node) == 2:
+        if len(node) <= 2:
             return dot, None
+        if node[0] == symbol.dotted_name:
+            pass
+        else:
             assert node[2][1] == 'as'
             assert node[3][0] == token.NAME
             return dot, node[3][1]
@@ -872,6 +898,20 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
+    def com_augassign_op(self, node):
+        assert node[0] == symbol.augassign
+        return node[1]
+
+    def com_augassign(self, node):
+        """Return node suitable for lvalue of augmented assignment
+
+        Names, slices, and attributes are the only allowable nodes.
+        """
+        l = self.com_node(node)
+        if l[0] in ('name', 'slice', 'subscript', 'getattr'):
+            return l
+        raise SyntaxError, "can't assign to %s" % l[0]
+
     def com_assign(self, node, assigning):
         # return a node suitable for use as an "lvalue"
         # loop to avoid trivial recursion
@@ -955,7 +995,6 @@ class Transformer:
         return Node(type, items)
 
     def com_stmt(self, node):
-        #pprint.pprint(node)
         result = self.com_node(node)
         try:
             result[0]
@@ -976,6 +1015,59 @@ class Transformer:
         else:
             stmts.append(result)
 
+    if hasattr(symbol, 'list_for'):
+        def com_list_constructor(self, nodelist):
+            # listmaker: test ( list_for | (',' test)* [','] )
+            values = [ ]
+            for i in range(1, len(nodelist)):
+                if nodelist[i][0] == symbol.list_for:
+                    assert len(nodelist[i:]) == 1
+                    return self.com_list_comprehension(values[0],
+                                                       nodelist[i])
+                elif nodelist[i][0] == token.COMMA:
+                    continue
+                values.append(self.com_node(nodelist[i]))
+            return Node('list', values)
+
+        def com_list_comprehension(self, expr, node):
+            # list_iter: list_for | list_if
+            # list_for: 'for' exprlist 'in' testlist [list_iter]
+            # list_if: 'if' test [list_iter]
+            lineno = node[1][2]
+            fors = []
+            while node:
+                if node[1][1] == 'for':
+                    assignNode = self.com_assign(node[2], OP_ASSIGN)
+                    listNode = self.com_node(node[4])
+                    newfor = Node('listcomp_for', assignNode,
+                                  listNode, [])
+                    newfor.lineno = node[1][2]
+                    fors.append(newfor)
+                    if len(node) == 5:
+                        node = None
+                    else:
+                        node = self.com_list_iter(node[5])
+                elif node[1][1] == 'if':
+                    test = self.com_node(node[2])
+                    newif = Node('listcomp_if', test)
+                    newif.lineno = node[1][2]
+                    newfor.ifs.append(newif)
+                    if len(node) == 3:
+                        node = None
+                    else:
+                        node = self.com_list_iter(node[3])
+                else:
+                    raise SyntaxError, \
+                          ("unexpected list comprehension element: %s %d"
+                           % (node, lineno))
+            n = Node('listcomp', expr, fors)
+            n.lineno = lineno
+            return n
+
+        def com_list_iter(self, node):
+            assert node[0] == symbol.list_iter
+            return node[1]
+    else:
         def com_list_constructor(self, nodelist):
             values = [ ]
             for i in range(1, len(nodelist), 2):
@@ -986,7 +1078,8 @@ class Transformer:
         # dictmaker: test ':' test (',' test ':' value)* [',']
         items = [ ]
         for i in range(1, len(nodelist), 4):
-            items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
+            items.append((self.com_node(nodelist[i]),
+                          self.com_node(nodelist[i+2])))
         return Node('dict', items)
 
     def com_apply_trailer(self, primaryNode, nodelist):
@@ -1250,3 +1343,21 @@ _assign_types = [
     symbol.term,
     symbol.factor,
     ]
+
+import types
+_names = {}
+for k, v in symbol.sym_name.items():
+    _names[k] = v
+for k, v in token.tok_name.items():
+    _names[k] = v
+
+def debug_tree(tree):
+    l = []
+    for elt in tree:
+        if type(elt) == types.IntType:
+            l.append(_names.get(elt, elt))
+        elif type(elt) == types.StringType:
+            l.append(elt)
+        else:
+            l.append(debug_tree(elt))
+    return l

Tools/compiler/compiler/ast.py

@@ -279,22 +279,24 @@ class Const(Node):
 class Print(Node):
   nodes['print'] = 'Print'
 
-  def __init__(self, nodes):
+  def __init__(self, nodes, dest):
     self.nodes = nodes
-    self._children = ('print', nodes)
+    self.dest = dest
+    self._children = ('print', nodes, dest)
 
   def __repr__(self):
-    return "Print(%s)" % self._children[1:]
+    return "Print(%s, %s)" % (self._children[1:-1], self._children[-1])
 
 class Printnl(Node):
   nodes['printnl'] = 'Printnl'
 
-  def __init__(self, nodes):
+  def __init__(self, nodes, dest):
     self.nodes = nodes
-    self._children = ('printnl', nodes)
+    self.dest = dest
+    self._children = ('printnl', nodes, dest)
 
   def __repr__(self):
-    return "Printnl(%s)" % self._children[1:]
+    return "Printnl(%s, %s)" % (self._children[1:-1], self._children[-1])
 
 class Discard(Node):
   nodes['discard'] = 'Discard'
@@ -306,6 +308,18 @@ class Discard(Node):
   def __repr__(self):
     return "Discard(%s)" % self._children[1:]
 
+class AugAssign(Node):
+  nodes['augassign'] = 'AugAssign'
+
+  def __init__(self, node, op, expr):
+    self.node = node
+    self.op = op
+    self.expr = expr
+    self._children = ('augassign', node, op, expr)
+
+  def __repr__(self):
+    return "AugAssign(%s)" % str(self._children[1:])
+
 class Assign(Node):
   nodes['assign'] = 'Assign'
 
@@ -360,6 +374,41 @@ class AssAttr(Node):
   def __repr__(self):
     return "AssAttr(%s,%s,%s)" % self._children[1:]
 
+class ListComp(Node):
+  nodes['listcomp'] = 'ListComp'
+
+  def __init__(self, expr, quals):
+    self.expr = expr
+    self.quals = quals
+    self._children = ('listcomp', expr, quals)
+
+  def __repr__(self):
+    return "ListComp(%s, %s)" % self._children[1:]
+
+class ListCompFor(Node):
+  nodes['listcomp_for'] = 'ListCompFor'
+
+  # transformer fills in ifs after node is created
+
+  def __init__(self, assign, list, ifs):
+    self.assign = assign
+    self.list = list
+    self.ifs = ifs
+    self._children = ('listcomp_for', assign, list, ifs)
+
+  def __repr__(self):
+    return "ListCompFor(%s, %s, %s)" % self._children[1:]
+
+class ListCompIf(Node):
+  nodes['listcomp_if'] = 'ListCompIf'
+
+  def __init__(self, test):
+    self.test = test
+    self._children = ('listcomp_if', test)
+
+  def __repr__(self):
+    return "ListCompIf(%s)" % self._children[1:]
+
 class List(Node):
   nodes['list'] = 'List'
 

Tools/compiler/compiler/pyassem.py

@@ -253,8 +253,14 @@ class PyFlowGraph(FlowGraph):
 
     def _lookupName(self, name, list):
         """Return index of name in list, appending if necessary"""
-        if name in list:
+        found = None
-            i = list.index(name)
+        t = type(name)
+        for i in range(len(list)):
+            # must do a comparison on type first to prevent UnicodeErrors 
+            if t == type(list[i]) and list[i] == name:
+                found = 1
+                break
+        if found:
             # this is cheap, but incorrect in some cases, e.g 2 vs. 2L
             if type(name) == type(list[i]):
                 return i

Tools/compiler/compiler/pycodegen.py

@@ -1,8 +1,10 @@
+import imp
 import os
 import marshal
 import stat
 import string
 import struct
+import sys
 import types
 from cStringIO import StringIO
 
@@ -10,6 +12,12 @@ from compiler import ast, parse, walk
 from compiler import pyassem, misc
 from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, TupleArg
 
+# Do we have Python 1.x or Python 2.x?
+try:
+    VERSION = sys.version_info[0]
+except AttributeError:
+    VERSION = 1
+
 callfunc_opcode_info = {
     # (Have *args, Have **args) : opcode
     (0,0) : "CALL_FUNCTION",
@@ -18,12 +26,12 @@ callfunc_opcode_info = {
     (1,1) : "CALL_FUNCTION_VAR_KW",
 }
 
-def compile(filename):
+def compile(filename, display=0):
     f = open(filename)
     buf = f.read()
     f.close()
     mod = Module(buf, filename)
-    mod.compile()
+    mod.compile(display)
     f = open(filename + "c", "wb")
     mod.dump(f)
     f.close()
@@ -34,28 +42,30 @@ class Module:
         self.source = source
         self.code = None
 
-    def compile(self):
+    def compile(self, display=0):
         ast = parse(self.source)
         root, filename = os.path.split(self.filename)
         gen = ModuleCodeGenerator(filename)
         walk(ast, gen, 1)
+        if display:
+            import pprint
+            print pprint.pprint(ast)
         self.code = gen.getCode()
 
     def dump(self, f):
         f.write(self.getPycHeader())
         marshal.dump(self.code, f)
 
-    MAGIC = (50823 | (ord('\r')<<16) | (ord('\n')<<24))
+    MAGIC = imp.get_magic()
 
     def getPycHeader(self):
         # compile.c uses marshal to write a long directly, with
         # calling the interface that would also generate a 1-byte code
         # to indicate the type of the value.  simplest way to get the
         # same effect is to call marshal and then skip the code.
-        magic = marshal.dumps(self.MAGIC)[1:]
         mtime = os.stat(self.filename)[stat.ST_MTIME]
         mtime = struct.pack('i', mtime)
-        return magic + mtime
+        return self.MAGIC + mtime
 
 class CodeGenerator:
 
@@ -63,7 +73,7 @@ class CodeGenerator:
 
     def __init__(self, filename):
 ## Subclasses must define a constructor that intializes self.graph
-## before calling this init function
+## before calling this init function, e.g.
 ##         self.graph = pyassem.PyFlowGraph()
         self.filename = filename
         self.locals = misc.Stack()
@@ -142,7 +152,6 @@ class CodeGenerator:
 
     def visitLambda(self, node):
         self._visitFuncOrLambda(node, isLambda=1)
-##        self.storeName("<lambda>")
 
     def _visitFuncOrLambda(self, node, isLambda):
         gen = FunctionCodeGenerator(node, self.filename, isLambda)
@@ -180,10 +189,6 @@ class CodeGenerator:
             test, suite = node.tests[i]
             self.set_lineno(test)
             self.visit(test)
-##            if i == numtests - 1 and not node.else_:
-##                nextTest = end
-##            else:
-##                nextTest = self.newBlock()
             nextTest = self.newBlock()
             self.emit('JUMP_IF_FALSE', nextTest)
             self.nextBlock()
@@ -304,6 +309,70 @@ class CodeGenerator:
             self.emit('POP_TOP')
             self.nextBlock(end)
 
+    # list comprehensions
+    __list_count = 0
+    
+    def visitListComp(self, node):
+        # XXX would it be easier to transform the AST into the form it
+        # would have if the list comp were expressed as a series of
+        # for and if stmts and an explicit append?
+        self.set_lineno(node)
+        # setup list
+        append = "$append%d" % self.__list_count
+        self.__list_count = self.__list_count + 1
+        self.emit('BUILD_LIST', 0)
+        self.emit('DUP_TOP')
+        self.emit('LOAD_ATTR', 'append')
+        self.storeName(append)
+        l = len(node.quals)
+        stack = []
+        for i, for_ in zip(range(l), node.quals):
+            start, anchor = self.visit(for_)
+            cont = None
+            for if_ in for_.ifs:
+                if cont is None:
+                    cont = self.newBlock()
+                self.visit(if_, cont)
+            stack.insert(0, (start, cont, anchor))
+            
+        self.loadName(append)
+        self.visit(node.expr)
+        self.emit('CALL_FUNCTION', 1)
+        self.emit('POP_TOP')
+        
+        for start, cont, anchor in stack:
+            if cont:
+                skip_one = self.newBlock()
+                self.emit('JUMP_FORWARD', skip_one)
+                self.nextBlock(cont)
+                self.emit('POP_TOP')
+                self.nextBlock(skip_one)
+            self.emit('JUMP_ABSOLUTE', start)
+            self.nextBlock(anchor)
+        self.delName(append)
+        
+        self.__list_count = self.__list_count - 1
+
+    def visitListCompFor(self, node):
+        self.set_lineno(node)
+        start = self.newBlock()
+        anchor = self.newBlock()
+
+        self.visit(node.list)
+        self.visit(ast.Const(0))
+        self.emit('SET_LINENO', node.lineno)
+        self.nextBlock(start)
+        self.emit('FOR_LOOP', anchor)
+        self.visit(node.assign)
+        return start, anchor
+
+    def visitListCompIf(self, node, branch):
+        self.set_lineno(node)
+        self.visit(node.test)
+        self.emit('JUMP_IF_FALSE', branch)
+        self.newBlock()
+        self.emit('POP_TOP')
+
     # exception related
 
     def visitAssert(self, node):
@@ -397,10 +466,6 @@ class CodeGenerator:
 
     # misc
 
-##     def visitStmt(self, node):
-##         # nothing to do except walk the children
-##         pass
-
     def visitDiscard(self, node):
         self.visit(node.expr)
         self.emit('POP_TOP')
@@ -426,17 +491,20 @@ class CodeGenerator:
     def visitImport(self, node):
         self.set_lineno(node)
         for name, alias in node.names:
+            if VERSION > 1:
                 self.emit('LOAD_CONST', None)
             self.emit('IMPORT_NAME', name)
-            self._resolveDots(name)
+            mod = string.split(name, ".")[0]
-            self.storeName(alias or name)
+            self.storeName(alias or mod)
 
     def visitFrom(self, node):
         self.set_lineno(node)
         fromlist = map(lambda (name, alias): name, node.names)
+        if VERSION > 1:
             self.emit('LOAD_CONST', tuple(fromlist))
         self.emit('IMPORT_NAME', node.modname)
         for name, alias in node.names:
+            if VERSION > 1:
                 if name == '*':
                     self.namespace = 0
                     self.emit('IMPORT_STAR')
@@ -447,6 +515,8 @@ class CodeGenerator:
                     self.emit('IMPORT_FROM', name)
                     self._resolveDots(name)
                     self.storeName(alias or name)
+            else:
+                self.emit('IMPORT_FROM', name)
         self.emit('POP_TOP')
 
     def _resolveDots(self, name):
@@ -491,13 +561,85 @@ class CodeGenerator:
             print "warning: unexpected flags:", node.flags
             print node
 
-    def visitAssTuple(self, node):
+    def _visitAssSequence(self, node, op='UNPACK_SEQUENCE'):
         if findOp(node) != 'OP_DELETE':
-            self.emit('UNPACK_SEQUENCE', len(node.nodes))
+            self.emit(op, len(node.nodes))
         for child in node.nodes:
             self.visit(child)
 
-    visitAssList = visitAssTuple
+    if VERSION > 1:
+        visitAssTuple = _visitAssSequence
+        visitAssList = _visitAssSequence
+    else:
+        def visitAssTuple(self, node):
+            self._visitAssSequence(node, 'UNPACK_TUPLE')
+
+        def visitAssList(self, node):
+            self._visitAssSequence(node, 'UNPACK_LIST')
+
+    # augmented assignment
+
+    def visitAugAssign(self, node):
+        aug_node = wrap_aug(node.node)
+        self.visit(aug_node, "load")
+        self.visit(node.expr)
+        self.emit(self._augmented_opcode[node.op])
+        self.visit(aug_node, "store")
+
+    _augmented_opcode = {
+        '+=' : 'INPLACE_ADD',
+        '-=' : 'INPLACE_SUBTRACT',
+        '*=' : 'INPLACE_MULTIPLY',
+        '/=' : 'INPLACE_DIVIDE',
+        '%=' : 'INPLACE_MODULO',
+        '**=': 'INPLACE_POWER',
+        '>>=': 'INPLACE_RSHIFT',
+        '<<=': 'INPLACE_LSHIFT',
+        '&=' : 'INPLACE_AND',
+        '^=' : 'INPLACE_XOR',
+        '|=' : 'INPLACE_OR',
+        }
+
+    def visitAugName(self, node, mode):
+        if mode == "load":
+            self.loadName(node.name)
+        elif mode == "store":
+            self.storeName(node.name)
+
+    def visitAugGetattr(self, node, mode):
+        if mode == "load":
+            self.visit(node.expr)
+            self.emit('DUP_TOP')
+            self.emit('LOAD_ATTR', node.attrname)
+        elif mode == "store":
+            self.emit('ROT_TWO')
+            self.emit('STORE_ATTR', node.attrname)
+
+    def visitAugSlice(self, node, mode):
+        if mode == "load":
+            self.visitSlice(node, 1)
+        elif mode == "store":
+            slice = 0
+            if node.lower:
+                slice = slice | 1
+            if node.upper:
+                slice = slice | 2
+            if slice == 0:
+                self.emit('ROT_TWO')
+            elif slice == 3:
+                self.emit('ROT_FOUR')
+            else:
+                self.emit('ROT_THREE')
+            self.emit('STORE_SLICE+%d' % slice)
+
+    def visitAugSubscript(self, node, mode):
+        if len(node.subs) > 1:
+            raise SyntaxError, "augmented assignment to tuple is not possible"
+        if mode == "load":
+            self.visitSubscript(node, 1)
+        elif mode == "store":
+            self.emit('ROT_THREE')
+            self.emit('STORE_SUBSCR')
 
     def visitExec(self, node):
         self.visit(node.expr)
@@ -533,12 +675,23 @@ class CodeGenerator:
 
     def visitPrint(self, node):
         self.set_lineno(node)
+        if node.dest:
+            self.visit(node.dest)
         for child in node.nodes:
+            if node.dest:
+                self.emit('DUP_TOP')
             self.visit(child)
+            if node.dest:
+                self.emit('ROT_TWO')
+                self.emit('PRINT_ITEM_TO')
+            else:
                 self.emit('PRINT_ITEM')
 
     def visitPrintnl(self, node):
         self.visitPrint(node)
+        if node.dest:
+            self.emit('PRINT_NEWLINE_TO')
+        else:
             self.emit('PRINT_NEWLINE')
 
     def visitReturn(self, node):
@@ -548,7 +701,8 @@ class CodeGenerator:
 
     # slice and subscript stuff
 
-    def visitSlice(self, node):
+    def visitSlice(self, node, aug_flag=None):
+        # aug_flag is used by visitAugSlice
         self.visit(node.expr)
         slice = 0
         if node.lower:
@@ -557,6 +711,13 @@ class CodeGenerator:
         if node.upper:
             self.visit(node.upper)
             slice = slice | 2
+        if aug_flag:
+            if slice == 0:
+                self.emit('DUP_TOP')
+            elif slice == 3:
+                self.emit('DUP_TOPX', 3)
+            else:
+                self.emit('DUP_TOPX', 2)
         if node.flags == 'OP_APPLY':
             self.emit('SLICE+%d' % slice)
         elif node.flags == 'OP_ASSIGN':
@@ -567,10 +728,12 @@ class CodeGenerator:
             print "weird slice", node.flags
             raise
 
-    def visitSubscript(self, node):
+    def visitSubscript(self, node, aug_flag=None):
         self.visit(node.expr)
         for sub in node.subs:
             self.visit(sub)
+        if aug_flag:
+            self.emit('DUP_TOPX', 2)
         if len(node.subs) > 1:
             self.emit('BUILD_TUPLE', len(node.subs))
         if node.flags == 'OP_APPLY':
@@ -740,7 +903,10 @@ class FunctionCodeGenerator(CodeGenerator):
                 self.unpackSequence(arg)
                         
     def unpackSequence(self, tup):
+        if VERSION > 1:
             self.emit('UNPACK_SEQUENCE', len(tup))
+        else:
+            self.emit('UNPACK_TUPLE', len(tup))
         for elt in tup:
             if type(elt) == types.TupleType:
                 self.unpackSequence(elt)
@@ -765,7 +931,6 @@ class ClassCodeGenerator(CodeGenerator):
         self.emit('LOAD_LOCALS')
         self.emit('RETURN_VALUE')
 
-
 def generateArgList(arglist):
     """Generate an arg list marking TupleArgs"""
     args = []
@@ -838,6 +1003,45 @@ class OpFinder:
         elif self.op != node.flags:
             raise ValueError, "mixed ops in stmt"
 
+class Delegator:
+    """Base class to support delegation for augmented assignment nodes
+
+    To generator code for augmented assignments, we use the following
+    wrapper classes.  In visitAugAssign, the left-hand expression node
+    is visited twice.  The first time the visit uses the normal method
+    for that node .  The second time the visit uses a different method
+    that generates the appropriate code to perform the assignment.
+    These delegator classes wrap the original AST nodes in order to
+    support the variant visit methods.
+    """
+    def __init__(self, obj):
+        self.obj = obj
+
+    def __getattr__(self, attr):
+        return getattr(self.obj, attr)
+
+class AugGetattr(Delegator):
+    pass
+
+class AugName(Delegator):
+    pass
+
+class AugSlice(Delegator):
+    pass
+
+class AugSubscript(Delegator):
+    pass
+
+wrapper = {
+    ast.Getattr: AugGetattr,
+    ast.Name: AugName,
+    ast.Slice: AugSlice,
+    ast.Subscript: AugSubscript,
+    }
+
+def wrap_aug(node):
+    return wrapper[node.__class__](node)
+
 if __name__ == "__main__":
     import sys
 

Tools/compiler/compiler/transformer.py

@@ -1,19 +1,3 @@
-#
-# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
-#
-# This module is provided under a BSD-ish license. See
-#   http://www.opensource.org/licenses/bsd-license.html
-# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
-#
-#
-# Written by Greg Stein (gstein@lyra.org)
-#        and Bill Tutt (rassilon@lima.mudlib.org)
-# February 1997.
-#
-# Support for ast.Node subclasses written and other revisions by
-#  Jeremy Hylton (jeremy@beopen.com)
-#
-
 """Parse tree transformation module.
 
 Transforms Python source code into an abstract syntax tree (AST)
@@ -24,7 +8,21 @@ parse(buf) -> AST
 parseFile(path) -> AST
 """
 
+# Original version written by Greg Stein (gstein@lyra.org)
+#                         and Bill Tutt (rassilon@lima.mudlib.org)
+# February 1997.
 #
+# Modifications and improvements for Python 2.0 by Jeremy Hylton and
+# Mark Hammond
+
+# Portions of this file are:
+# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
+#
+# This module is provided under a BSD-ish license. See
+#   http://www.opensource.org/licenses/bsd-license.html
+# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
+
+
 # The output tree has the following nodes:
 #
 # Source Python line #'s appear at the end of each of all of these nodes
@@ -49,9 +47,10 @@ parseFile(path) -> AST
 # tryexcept:  trySuiteNode, [ (exprNode, assgnNode, suiteNode), ... ], elseNode
 # return:     valueNode
 # const:      value
-# print:      [ node1, ..., nodeN ]
+# print:      [ node1, ..., nodeN ] [, dest]
-# printnl:    [ node1, ..., nodeN ]
+# printnl:    [ node1, ..., nodeN ] [, dest]
 # discard:    exprNode
+# augassign:  node, op, expr
 # assign:     [ node1, ..., nodeN ], exprNode
 # ass_tuple:  [ node1, ..., nodeN ]
 # ass_list:   [ node1, ..., nodeN ]
@@ -97,12 +96,12 @@ parseFile(path) -> AST
 
 import ast
 import parser
+# Care must be taken to use only symbols and tokens defined in Python
+# 1.5.2 for code branches executed in 1.5.2
 import symbol
 import token
 import string
 
-import pprint
-
 error = 'walker.error'
 
 from consts import CO_VARARGS, CO_VARKEYWORDS
@@ -328,27 +327,44 @@ class Transformer:
     #
 
     def expr_stmt(self, nodelist):
-        # testlist ('=' testlist)*
+        # augassign testlist | testlist ('=' testlist)*
         exprNode = self.com_node(nodelist[-1])
         if len(nodelist) == 1:
             return Node('discard', exprNode)
+        if nodelist[1][0] == token.EQUAL:
             nodes = [ ]
             for i in range(0, len(nodelist) - 2, 2):
                 nodes.append(self.com_assign(nodelist[i], OP_ASSIGN))
             n = Node('assign', nodes, exprNode)
             n.lineno = nodelist[1][2]
+        else:
+            lval = self.com_augassign(nodelist[0])
+            op = self.com_augassign_op(nodelist[1])
+            n = Node('augassign', lval, op[1], exprNode)
+            n.lineno = op[2]
         return n
 
     def print_stmt(self, nodelist):
-        # print: (test ',')* [test]
+        # print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
         items = [ ]
-        for i in range(1, len(nodelist), 2):
+        if len(nodelist) == 1:
+            start = 1
+            dest = None
+        elif nodelist[1][0] == token.RIGHTSHIFT:
+            assert len(nodelist) == 3 \
+                   or nodelist[3][0] == token.COMMA
+            dest = self.com_node(nodelist[2])
+            start = 4
+        else:
+            dest = None
+            start = 1
+        for i in range(start, len(nodelist), 2):
             items.append(self.com_node(nodelist[i]))
         if nodelist[-1][0] == token.COMMA:
-            n = Node('print', items)
+            n = Node('print', items, dest)
             n.lineno = nodelist[0][2]
             return n
-        n = Node('printnl', items)
+        n = Node('printnl', items, dest)
         n.lineno = nodelist[0][2]
         return n
 
@@ -405,15 +421,22 @@ class Transformer:
         # import_stmt: 'import' dotted_as_name (',' dotted_as_name)* |
         # from: 'from' dotted_name 'import'
         #                        ('*' | import_as_name (',' import_as_name)*)
-        names = []
-        is_as = 0
         if nodelist[0][1] == 'from':
+            names = []
+            if nodelist[3][0] == token.NAME:
+                for i in range(3, len(nodelist), 2):
+                    names.append((nodelist[i][1], None))
+            else:
                 for i in range(3, len(nodelist), 2):
                     names.append(self.com_import_as_name(nodelist[i][1]))
             n = Node('from', self.com_dotted_name(nodelist[1]), names)
             n.lineno = nodelist[0][2]
             return n
 
+        if nodelist[1][0] == symbol.dotted_name:
+            names = [(self.com_dotted_name(nodelist[1][1:]), None)]
+        else:
+            names = []
             for i in range(1, len(nodelist), 2):
                 names.append(self.com_dotted_as_name(nodelist[i]))
         n = Node('import', names)
@@ -737,7 +760,7 @@ class Transformer:
             return Node('discard', Node('const', None))
 
         if node[0] not in _legal_node_types:
-            raise error, 'illegal node passed to com_node: %s' % node[0]
+            raise error, 'illegal node passed to com_node: %s' % `node`
 
 #    print "dispatch", self._dispatch[node[0]].__name__, node
         return self._dispatch[node[0]](node[1:])
@@ -818,8 +841,11 @@ class Transformer:
 
     def com_dotted_as_name(self, node):
         dot = self.com_dotted_name(node[1])
-        if len(node) == 2:
+        if len(node) <= 2:
             return dot, None
+        if node[0] == symbol.dotted_name:
+            pass
+        else:
             assert node[2][1] == 'as'
             assert node[3][0] == token.NAME
             return dot, node[3][1]
@@ -872,6 +898,20 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
+    def com_augassign_op(self, node):
+        assert node[0] == symbol.augassign
+        return node[1]
+
+    def com_augassign(self, node):
+        """Return node suitable for lvalue of augmented assignment
+
+        Names, slices, and attributes are the only allowable nodes.
+        """
+        l = self.com_node(node)
+        if l[0] in ('name', 'slice', 'subscript', 'getattr'):
+            return l
+        raise SyntaxError, "can't assign to %s" % l[0]
+
     def com_assign(self, node, assigning):
         # return a node suitable for use as an "lvalue"
         # loop to avoid trivial recursion
@@ -955,7 +995,6 @@ class Transformer:
         return Node(type, items)
 
     def com_stmt(self, node):
-        #pprint.pprint(node)
         result = self.com_node(node)
         try:
             result[0]
@@ -976,6 +1015,59 @@ class Transformer:
         else:
             stmts.append(result)
 
+    if hasattr(symbol, 'list_for'):
+        def com_list_constructor(self, nodelist):
+            # listmaker: test ( list_for | (',' test)* [','] )
+            values = [ ]
+            for i in range(1, len(nodelist)):
+                if nodelist[i][0] == symbol.list_for:
+                    assert len(nodelist[i:]) == 1
+                    return self.com_list_comprehension(values[0],
+                                                       nodelist[i])
+                elif nodelist[i][0] == token.COMMA:
+                    continue
+                values.append(self.com_node(nodelist[i]))
+            return Node('list', values)
+
+        def com_list_comprehension(self, expr, node):
+            # list_iter: list_for | list_if
+            # list_for: 'for' exprlist 'in' testlist [list_iter]
+            # list_if: 'if' test [list_iter]
+            lineno = node[1][2]
+            fors = []
+            while node:
+                if node[1][1] == 'for':
+                    assignNode = self.com_assign(node[2], OP_ASSIGN)
+                    listNode = self.com_node(node[4])
+                    newfor = Node('listcomp_for', assignNode,
+                                  listNode, [])
+                    newfor.lineno = node[1][2]
+                    fors.append(newfor)
+                    if len(node) == 5:
+                        node = None
+                    else:
+                        node = self.com_list_iter(node[5])
+                elif node[1][1] == 'if':
+                    test = self.com_node(node[2])
+                    newif = Node('listcomp_if', test)
+                    newif.lineno = node[1][2]
+                    newfor.ifs.append(newif)
+                    if len(node) == 3:
+                        node = None
+                    else:
+                        node = self.com_list_iter(node[3])
+                else:
+                    raise SyntaxError, \
+                          ("unexpected list comprehension element: %s %d"
+                           % (node, lineno))
+            n = Node('listcomp', expr, fors)
+            n.lineno = lineno
+            return n
+
+        def com_list_iter(self, node):
+            assert node[0] == symbol.list_iter
+            return node[1]
+    else:
         def com_list_constructor(self, nodelist):
             values = [ ]
             for i in range(1, len(nodelist), 2):
@@ -986,7 +1078,8 @@ class Transformer:
         # dictmaker: test ':' test (',' test ':' value)* [',']
         items = [ ]
         for i in range(1, len(nodelist), 4):
-            items.append((self.com_node(nodelist[i]), self.com_node(nodelist[i+2])))
+            items.append((self.com_node(nodelist[i]),
+                          self.com_node(nodelist[i+2])))
         return Node('dict', items)
 
     def com_apply_trailer(self, primaryNode, nodelist):
@@ -1250,3 +1343,21 @@ _assign_types = [
     symbol.term,
     symbol.factor,
     ]
+
+import types
+_names = {}
+for k, v in symbol.sym_name.items():
+    _names[k] = v
+for k, v in token.tok_name.items():
+    _names[k] = v
+
+def debug_tree(tree):
+    l = []
+    for elt in tree:
+        if type(elt) == types.IntType:
+            l.append(_names.get(elt, elt))
+        elif type(elt) == types.StringType:
+            l.append(elt)
+        else:
+            l.append(debug_tree(elt))
+    return l