patches from Mark Hammond

Attached is a set of diffs for the .py compiler that adds support
for the new extended call syntax.

compiler/ast.py:
CallFunc node gets 2 new children to support extended call syntax -
"star_args" (for "*args") and "dstar_args" (for "**args")

compiler/pyassem.py
It appear that self.lnotab is supposed to be responsible for
tracking line numbers, but self.firstlineno was still hanging
around.  Removed self.firstlineno completely.  NOTE - I didnt
actually test that the generated code has the correct line numbers!!

Stack depth tracking appeared a little broken - the checks never
made it beyond the "self.patterns" check - thus, the custom methods
were never called!  Fixed this.

(XXX Jeremy notes: I think this code is still broken because it
doesn't track stack effects across block bounaries.)

Added support for the new extended call syntax opcodes for depth
calculations.

compiler/pycodegen.py

Added support for the new extended call syntax opcodes.

compiler/transformer.py

Added support for the new extended call syntax.

Lib/compiler/ast.py

@@ -445,13 +445,15 @@ class Getattr(Node):
 class CallFunc(Node):
   nodes['call_func'] = 'CallFunc'
 
-  def __init__(self, node, args):
+  def __init__(self, node, args, star_args = None, dstar_args = None):
     self.node = node
     self.args = args
-    self._children = ('call_func', node, args)
+    self.star_args = star_args
+    self.dstar_args = dstar_args
+    self._children = ('call_func', node, args, star_args, dstar_args)
 
   def __repr__(self):
-    return "CallFunc(%s,%s)" % self._children[1:]
+    return "CallFunc(%s,%s,*%s, **%s)" % self._children[1:]
 
 class Keyword(Node):
   nodes['keyword'] = 'Keyword'

Lib/compiler/pyassem.py

@@ -149,7 +149,6 @@ class PyFlowGraph(FlowGraph):
 	    self.flags = CO_OPTIMIZED | CO_NEWLOCALS 
 	else:
 	    self.flags = 0
-	self.firstlineno = None
 	self.consts = []
 	self.names = []
         self.varnames = list(args) or []
@@ -314,8 +313,6 @@ class PyFlowGraph(FlowGraph):
                 oparg = t[1]
                 if opname == "SET_LINENO":
                     lnotab.nextLine(oparg)
-                    if self.firstlineno is None:
-                        self.firstlineno = oparg
                 hi, lo = twobyte(oparg)
 		try:
 		    lnotab.addCode(self.opnum[opname], lo, hi)
@@ -342,7 +339,7 @@ class PyFlowGraph(FlowGraph):
         return new.code(argcount, nlocals, self.stacksize, self.flags,
                         self.lnotab.getCode(), self.getConsts(),
                         tuple(self.names), tuple(self.varnames),
-                        self.filename, self.name, self.firstlineno,
+                        self.filename, self.name, self.lnotab.firstline,
                         self.lnotab.getTable())
 
     def getConsts(self):
@@ -464,14 +461,16 @@ class StackDepthTracker:
                 if depth > maxDepth:
                     maxDepth = depth
                 # now check patterns
-                for pat, delta in self.patterns:
+                for pat, pat_delta in self.patterns:
                     if opname[:len(pat)] == pat:
+                        delta = pat_delta
                         depth = depth + delta
                         break
                 # if we still haven't found a match
                 if delta == 0:
-                    meth = getattr(self, opname)
+                    meth = getattr(self, opname, None)
-                    depth = depth + meth(i[1])
+                    if meth is not None:
+                        depth = depth + meth(i[1])
             if depth < 0:
                 depth = 0
         return maxDepth
@@ -527,6 +526,12 @@ class StackDepthTracker:
     def CALL_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
         return lo + hi * 2
+    def CALL_FUNCTION_VAR(self, argc):
+        return self.CALL_FUNCTION(argc)+1
+    def CALL_FUNCTION_KW(self, argc):
+        return self.CALL_FUNCTION(argc)+1
+    def CALL_FUNCTION_VAR_KW(self, argc):
+        return self.CALL_FUNCTION(argc)+2
     def MAKE_FUNCTION(self, argc):
         return -argc
     def BUILD_SLICE(self, argc):

Lib/compiler/pycodegen.py

@@ -9,6 +9,14 @@ from compiler import ast, parse, walk
 from compiler import pyassem, misc
 from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, TupleArg
 
+callfunc_opcode_info = {
+    # (Have *args, Have **args) : opcode
+    (0,0) : "CALL_FUNCTION",
+    (1,0) : "CALL_FUNCTION_VAR",
+    (0,1) : "CALL_FUNCTION_KW",
+    (1,1) : "CALL_FUNCTION_VAR_KW",
+}
+
 def compile(filename):
     f = open(filename)
     buf = f.read()
@@ -478,7 +486,14 @@ class CodeGenerator:
                 kw = kw + 1
             else:
                 pos = pos + 1
-        self.emit('CALL_FUNCTION', kw << 8 | pos)
+        if node.star_args is not None:
+            self.visit(node.star_args)
+        if node.dstar_args is not None:
+            self.visit(node.dstar_args)
+        have_star = node.star_args is not None
+        have_dstar = node.dstar_args is not None
+        opcode = callfunc_opcode_info[have_star, have_dstar]
+        self.emit(opcode, kw << 8 | pos)
 
     def visitPrint(self, node):
         self.emit('SET_LINENO', node.lineno)

Lib/compiler/transformer.py

@@ -984,10 +984,32 @@ class Transformer:
       return Node('call_func', primaryNode, [ ])
     args = [ ]
     kw = 0
-    for i in range(1, len(nodelist), 2):
+    len_nodelist = len(nodelist)
-      kw, result = self.com_argument(nodelist[i], kw)
+    for i in range(1, len_nodelist, 2):
+      node = nodelist[i]
+      if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+          break
+      kw, result = self.com_argument(node, kw)
       args.append(result)
-    return Node('call_func', primaryNode, args)
+    else:
+        i = i + 1 # No broken by star arg, so skip the last one we processed.
+    star_node = dstar_node = None
+    while i < len_nodelist:
+        tok = nodelist[i]
+        ch = nodelist[i+1]
+        i = i + 3
+        if tok[0]==token.STAR:
+            if star_node is not None:
+                raise SyntaxError, 'already have the varargs indentifier'
+            star_node = self.com_node(ch)
+        elif tok[0]==token.DOUBLESTAR:
+            if dstar_node is not None:
+                raise SyntaxError, 'already have the kwargs indentifier'
+            dstar_node = self.com_node(ch)
+        else:
+            raise SyntaxError, 'unknown node type: %s' % tok
+
+    return Node('call_func', primaryNode, args, star_node, dstar_node)
 
   def com_argument(self, nodelist, kw):
     if len(nodelist) == 2:

Tools/compiler/compiler/ast.py

@@ -445,13 +445,15 @@ class Getattr(Node):
 class CallFunc(Node):
   nodes['call_func'] = 'CallFunc'
 
-  def __init__(self, node, args):
+  def __init__(self, node, args, star_args = None, dstar_args = None):
     self.node = node
     self.args = args
-    self._children = ('call_func', node, args)
+    self.star_args = star_args
+    self.dstar_args = dstar_args
+    self._children = ('call_func', node, args, star_args, dstar_args)
 
   def __repr__(self):
-    return "CallFunc(%s,%s)" % self._children[1:]
+    return "CallFunc(%s,%s,*%s, **%s)" % self._children[1:]
 
 class Keyword(Node):
   nodes['keyword'] = 'Keyword'

Tools/compiler/compiler/pyassem.py

@@ -149,7 +149,6 @@ class PyFlowGraph(FlowGraph):
 	    self.flags = CO_OPTIMIZED | CO_NEWLOCALS 
 	else:
 	    self.flags = 0
-	self.firstlineno = None
 	self.consts = []
 	self.names = []
         self.varnames = list(args) or []
@@ -314,8 +313,6 @@ class PyFlowGraph(FlowGraph):
                 oparg = t[1]
                 if opname == "SET_LINENO":
                     lnotab.nextLine(oparg)
-                    if self.firstlineno is None:
-                        self.firstlineno = oparg
                 hi, lo = twobyte(oparg)
 		try:
 		    lnotab.addCode(self.opnum[opname], lo, hi)
@@ -342,7 +339,7 @@ class PyFlowGraph(FlowGraph):
         return new.code(argcount, nlocals, self.stacksize, self.flags,
                         self.lnotab.getCode(), self.getConsts(),
                         tuple(self.names), tuple(self.varnames),
-                        self.filename, self.name, self.firstlineno,
+                        self.filename, self.name, self.lnotab.firstline,
                         self.lnotab.getTable())
 
     def getConsts(self):
@@ -464,14 +461,16 @@ class StackDepthTracker:
                 if depth > maxDepth:
                     maxDepth = depth
                 # now check patterns
-                for pat, delta in self.patterns:
+                for pat, pat_delta in self.patterns:
                     if opname[:len(pat)] == pat:
+                        delta = pat_delta
                         depth = depth + delta
                         break
                 # if we still haven't found a match
                 if delta == 0:
-                    meth = getattr(self, opname)
+                    meth = getattr(self, opname, None)
-                    depth = depth + meth(i[1])
+                    if meth is not None:
+                        depth = depth + meth(i[1])
             if depth < 0:
                 depth = 0
         return maxDepth
@@ -527,6 +526,12 @@ class StackDepthTracker:
     def CALL_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
         return lo + hi * 2
+    def CALL_FUNCTION_VAR(self, argc):
+        return self.CALL_FUNCTION(argc)+1
+    def CALL_FUNCTION_KW(self, argc):
+        return self.CALL_FUNCTION(argc)+1
+    def CALL_FUNCTION_VAR_KW(self, argc):
+        return self.CALL_FUNCTION(argc)+2
     def MAKE_FUNCTION(self, argc):
         return -argc
     def BUILD_SLICE(self, argc):

Tools/compiler/compiler/pycodegen.py

@@ -9,6 +9,14 @@ from compiler import ast, parse, walk
 from compiler import pyassem, misc
 from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, TupleArg
 
+callfunc_opcode_info = {
+    # (Have *args, Have **args) : opcode
+    (0,0) : "CALL_FUNCTION",
+    (1,0) : "CALL_FUNCTION_VAR",
+    (0,1) : "CALL_FUNCTION_KW",
+    (1,1) : "CALL_FUNCTION_VAR_KW",
+}
+
 def compile(filename):
     f = open(filename)
     buf = f.read()
@@ -478,7 +486,14 @@ class CodeGenerator:
                 kw = kw + 1
             else:
                 pos = pos + 1
-        self.emit('CALL_FUNCTION', kw << 8 | pos)
+        if node.star_args is not None:
+            self.visit(node.star_args)
+        if node.dstar_args is not None:
+            self.visit(node.dstar_args)
+        have_star = node.star_args is not None
+        have_dstar = node.dstar_args is not None
+        opcode = callfunc_opcode_info[have_star, have_dstar]
+        self.emit(opcode, kw << 8 | pos)
 
     def visitPrint(self, node):
         self.emit('SET_LINENO', node.lineno)

Tools/compiler/compiler/transformer.py

@@ -984,10 +984,32 @@ class Transformer:
       return Node('call_func', primaryNode, [ ])
     args = [ ]
     kw = 0
-    for i in range(1, len(nodelist), 2):
+    len_nodelist = len(nodelist)
-      kw, result = self.com_argument(nodelist[i], kw)
+    for i in range(1, len_nodelist, 2):
+      node = nodelist[i]
+      if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+          break
+      kw, result = self.com_argument(node, kw)
       args.append(result)
-    return Node('call_func', primaryNode, args)
+    else:
+        i = i + 1 # No broken by star arg, so skip the last one we processed.
+    star_node = dstar_node = None
+    while i < len_nodelist:
+        tok = nodelist[i]
+        ch = nodelist[i+1]
+        i = i + 3
+        if tok[0]==token.STAR:
+            if star_node is not None:
+                raise SyntaxError, 'already have the varargs indentifier'
+            star_node = self.com_node(ch)
+        elif tok[0]==token.DOUBLESTAR:
+            if dstar_node is not None:
+                raise SyntaxError, 'already have the kwargs indentifier'
+            dstar_node = self.com_node(ch)
+        else:
+            raise SyntaxError, 'unknown node type: %s' % tok
+
+    return Node('call_func', primaryNode, args, star_node, dstar_node)
 
   def com_argument(self, nodelist, kw):
     if len(nodelist) == 2: