Merged revisions 55270-55324 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/p3yk

........
  r55271 | fred.drake | 2007-05-11 10:14:47 -0700 (Fri, 11 May 2007) | 3 lines

  remove jpeg, panel libraries for SGI; there is more IRIX stuff left over,
  I guess that should be removed too, but will leave for someone who is sure
........
  r55280 | fred.drake | 2007-05-11 19:11:37 -0700 (Fri, 11 May 2007) | 1 line

  remove mention of file that has been removed
........
  r55301 | brett.cannon | 2007-05-13 17:38:05 -0700 (Sun, 13 May 2007) | 4 lines

  Remove rexec and Bastion from the stdlib.  This also eliminates the need for
  f_restricted on frames.  This in turn negates the need for
  PyEval_GetRestricted() and PyFrame_IsRestricted().
........
  r55303 | brett.cannon | 2007-05-13 19:22:22 -0700 (Sun, 13 May 2007) | 2 lines

  Remove the md5 and sha modules.
........
  r55305 | george.yoshida | 2007-05-13 19:45:55 -0700 (Sun, 13 May 2007) | 2 lines

  fix markup
........
  r55306 | neal.norwitz | 2007-05-13 19:47:57 -0700 (Sun, 13 May 2007) | 1 line

  Get the doc building again after some removals.
........
  r55307 | neal.norwitz | 2007-05-13 19:50:45 -0700 (Sun, 13 May 2007) | 1 line

  Get test_pyclbr passing again after getstatus was removed from commands.  This "test case" was weird since it was just importing a seemingly random module.  Remove the import
........
  r55322 | brett.cannon | 2007-05-14 14:09:20 -0700 (Mon, 14 May 2007) | 3 lines

  Remove the compiler package.  Will eventually need a mechanism to byte compile
  an AST.
........
This commit is contained in:
Guido van Rossum 2007-05-14 22:03:55 +00:00
parent 827b055ffe
commit a8add0ec5e
68 changed files with 52 additions and 9866 deletions

View file

@ -1,8 +0,0 @@
Fred L. Drake, Jr.
Mark Hammond
Shane Hathaway
Neil Schemenauer
Evan Simpson
Greg Stein
Bill Tutt
Moshe Zadka

View file

@ -1,18 +0,0 @@
This directory contains support tools for the Python compiler package,
which is now part of the standard library.
compile.py Demo that compiles a Python module into a .pyc file
using the pure-Python compiler code.
demo.py Prints the names of all the methods defined in a module,
as a demonstration of walking through the abstract syntax
tree produced by the parser.
dumppyc.py Dumps the contents of a .pyc file, printing
the attributes of the code object followed by a
code disassembly.
regrtest.py Runs the Python test suite using bytecode generated
by the pure-Python compiler code instead of the
builtin compiler.

View file

@ -1,105 +0,0 @@
# This file describes the nodes of the AST in ast.py. The module is
# generated by astgen.py.
# The descriptions use the following special notation to describe
# properties of the children:
# * this child is not a node
# ! this child is a sequence that contains nodes in it
# & this child may be set to None
# = ... a default value for the node constructor (optional args)
#
# If you add node types here, please be sure to update the list of
# Node types in Doc/lib/asttable.tex.
Module: doc*, node
Stmt: nodes!
Decorators: nodes!
Function: decorators&, name*, arguments!, defaults!, kwonlyargs!, returns&, flags*, doc*, code
Lambda: arguments!, defaults!, kwonlyargs!, flags*, code
SimpleArg: name*, annotation&
NestedArgs: args!
Kwarg: arg, expr
Class: name*, bases!, doc*, code
Pass:
Break:
Continue:
For: assign, list, body, else_&
While: test, body, else_&
With: expr, vars&, body
If: tests!, else_&
IfExp: test, then, else_
From: modname*, names*, level*
Import: names*
Raise: expr1&, expr2&, expr3&
TryFinally: body, final
TryExcept: body, handlers!, else_&
Return: value
Yield: value
Const: value*
Discard: expr
AugAssign: node, op*, expr
Assign: nodes!, expr
AssTuple: nodes!
AssList: nodes!
AssName: name*, flags*
AssAttr: expr, attrname*, flags*
ListComp: expr, quals!
ListCompFor: assign, list, ifs!
ListCompIf: test
GenExpr: code
GenExprInner: expr, quals!
GenExprFor: assign, iter, ifs!
GenExprIf: test
List: nodes!
Dict: items!
Set: items!
Not: expr
Compare: expr, ops!
Name: name*
Global: names*
Getattr: expr, attrname*
CallFunc: node, args!, star_args& = None, dstar_args& = None
Keyword: name*, expr
Subscript: expr, flags*, subs!
Sliceobj: nodes!
Slice: expr, flags*, lower&, upper&
Assert: test, fail&
Tuple: nodes!
Or: nodes!
And: nodes!
Bitor: nodes!
Bitxor: nodes!
Bitand: nodes!
LeftShift: (left, right)
RightShift: (left, right)
Add: (left, right)
Sub: (left, right)
Mul: (left, right)
Div: (left, right)
Mod: (left, right)
Power: (left, right)
FloorDiv: (left, right)
UnaryAdd: expr
UnarySub: expr
Invert: expr
init(Function):
self.varargs = self.kwargs = None
if flags & CO_VARARGS:
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
init(Lambda):
self.varargs = self.kwargs = None
if flags & CO_VARARGS:
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
self.returns = None
init(GenExpr):
self.arguments = [SimpleArg('.0', None)]
self.varargs = self.kwargs = None
self.kwonlyargs = ()
init(GenExprFor):
self.is_outmost = False

View file

@ -1,292 +0,0 @@
"""Generate ast module from specification
This script generates the ast module from a simple specification,
which makes it easy to accomodate changes in the grammar. This
approach would be quite reasonable if the grammar changed often.
Instead, it is rather complex to generate the appropriate code. And
the Node interface has changed more often than the grammar.
"""
import fileinput
import getopt
import re
import sys
from StringIO import StringIO
SPEC = "ast.txt"
COMMA = ", "
def load_boilerplate(file):
f = open(file)
buf = f.read()
f.close()
i = buf.find('### ''PROLOGUE')
j = buf.find('### ''EPILOGUE')
pro = buf[i+12:j].strip()
epi = buf[j+12:].strip()
return pro, epi
def strip_default(arg):
"""Return the argname from an 'arg = default' string"""
i = arg.find('=')
if i == -1:
return arg
t = arg[:i].strip()
return t
P_NODE = 1
P_OTHER = 2
P_NESTED = 3
P_NONE = 4
class NodeInfo:
"""Each instance describes a specific AST node"""
def __init__(self, name, args):
self.name = name
self.args = args.strip()
self.argnames = self.get_argnames()
self.argprops = self.get_argprops()
self.nargs = len(self.argnames)
self.init = []
def get_argnames(self):
if '(' in self.args:
i = self.args.find('(')
j = self.args.rfind(')')
args = self.args[i+1:j]
else:
args = self.args
return [strip_default(arg.strip())
for arg in args.split(',') if arg]
def get_argprops(self):
"""Each argument can have a property like '*' or '!'
XXX This method modifies the argnames in place!
"""
d = {}
hardest_arg = P_NODE
for i in range(len(self.argnames)):
arg = self.argnames[i]
if arg.endswith('*'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_OTHER
hardest_arg = max(hardest_arg, P_OTHER)
elif arg.endswith('!'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NESTED
hardest_arg = max(hardest_arg, P_NESTED)
elif arg.endswith('&'):
arg = self.argnames[i] = arg[:-1]
d[arg] = P_NONE
hardest_arg = max(hardest_arg, P_NONE)
else:
d[arg] = P_NODE
self.hardest_arg = hardest_arg
if hardest_arg > P_NODE:
self.args = self.args.replace('*', '')
self.args = self.args.replace('!', '')
self.args = self.args.replace('&', '')
return d
def gen_source(self):
buf = StringIO()
print("class %s(Node):" % self.name, file=buf)
self._gen_init(buf)
print(file=buf)
self._gen_getChildren(buf)
print(file=buf)
self._gen_getChildNodes(buf)
print(file=buf)
self._gen_repr(buf)
buf.seek(0, 0)
return buf.read()
def _gen_init(self, buf):
if self.args:
print(" def __init__(self, %s, lineno=None):" % self.args, file=buf)
else:
print(" def __init__(self, lineno=None):", file=buf)
if self.argnames:
for name in self.argnames:
print(" self.%s = %s" % (name, name), file=buf)
print(" self.lineno = lineno", file=buf)
# Copy the lines in self.init, indented four spaces. The rstrip()
# business is to get rid of the four spaces if line happens to be
# empty, so that reindent.py is happy with the output.
for line in self.init:
print((" " + line).rstrip(), file=buf)
def _gen_getChildren(self, buf):
print(" def getChildren(self):", file=buf)
if len(self.argnames) == 0:
print(" return ()", file=buf)
else:
if self.hardest_arg < P_NESTED:
clist = COMMA.join(["self.%s" % c
for c in self.argnames])
if self.nargs == 1:
print(" return %s," % clist, file=buf)
else:
print(" return %s" % clist, file=buf)
else:
if len(self.argnames) == 1:
print(" return tuple(flatten(self.%s))" % self.argnames[0], file=buf)
else:
print(" children = []", file=buf)
template = " children.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NESTED:
print(template % ("extend", "flatten(",
name, ")"), file=buf)
else:
print(template % ("append", "", name, ""), file=buf)
print(" return tuple(children)", file=buf)
def _gen_getChildNodes(self, buf):
print(" def getChildNodes(self):", file=buf)
if len(self.argnames) == 0:
print(" return ()", file=buf)
else:
if self.hardest_arg < P_NESTED:
clist = ["self.%s" % c
for c in self.argnames
if self.argprops[c] == P_NODE]
if len(clist) == 0:
print(" return ()", file=buf)
elif len(clist) == 1:
print(" return %s," % clist[0], file=buf)
else:
print(" return %s" % COMMA.join(clist), file=buf)
else:
print(" nodelist = []", file=buf)
template = " nodelist.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
tmp = (" if self.%s is not None:\n"
" nodelist.append(self.%s)")
print(tmp % (name, name), file=buf)
elif self.argprops[name] == P_NESTED:
print(template % ("extend", "flatten_nodes(",
name, ")"), file=buf)
elif self.argprops[name] == P_NODE:
print(template % ("append", "", name, ""), file=buf)
print(" return tuple(nodelist)", file=buf)
def _gen_repr(self, buf):
print(" def __repr__(self):", file=buf)
if self.argnames:
fmt = COMMA.join(["%s"] * self.nargs)
if '(' in self.args:
fmt = '(%s)' % fmt
vals = ["repr(self.%s)" % name for name in self.argnames]
vals = COMMA.join(vals)
if self.nargs == 1:
vals = vals + ","
print(' return "%s(%s)" %% (%s)' % \
(self.name, fmt, vals), file=buf)
else:
print(' return "%s()"' % self.name, file=buf)
rx_init = re.compile('init\((.*)\):')
def parse_spec(file):
classes = {}
cur = None
for line in fileinput.input(file):
if line.strip().startswith('#'):
continue
mo = rx_init.search(line)
if mo is None:
if cur is None:
# a normal entry
try:
name, args = line.split(':')
except ValueError:
continue
classes[name] = NodeInfo(name, args)
cur = None
else:
# some code for the __init__ method
cur.init.append(line)
else:
# some extra code for a Node's __init__ method
name = mo.group(1)
cur = classes[name]
return sorted(classes.values(), key=lambda n: n.name)
def main():
prologue, epilogue = load_boilerplate(sys.argv[-1])
print(prologue)
print()
classes = parse_spec(SPEC)
for info in classes:
print(info.gen_source())
print(epilogue)
if __name__ == "__main__":
main()
sys.exit(0)
### PROLOGUE
"""Python abstract syntax node definitions
This file is automatically generated by Tools/compiler/astgen.py
"""
from compiler.consts import CO_VARARGS, CO_VARKEYWORDS
def flatten(seq):
l = []
for elt in seq:
t = type(elt)
if t is tuple or t is list:
for elt2 in flatten(elt):
l.append(elt2)
else:
l.append(elt)
return l
def flatten_nodes(seq):
return [n for n in flatten(seq) if isinstance(n, Node)]
nodes = {}
class Node:
"""Abstract base class for ast nodes."""
def getChildren(self):
pass # implemented by subclasses
def __iter__(self):
for n in self.getChildren():
yield n
def asList(self): # for backwards compatibility
return self.getChildren()
def getChildNodes(self):
pass # implemented by subclasses
class EmptyNode(Node):
def getChildNodes(self):
return ()
def getChildren(self):
return ()
class Expression(Node):
# Expression is an artificial node class to support "eval"
nodes["expression"] = "Expression"
def __init__(self, node):
self.node = node
def getChildren(self):
return self.node,
def getChildNodes(self):
return self.node,
def __repr__(self):
return "Expression(%s)" % (repr(self.node))
### EPILOGUE
for name, obj in globals().items():
if isinstance(obj, type) and issubclass(obj, Node):
nodes[name.lower()] = obj

View file

@ -1,51 +0,0 @@
import sys
import getopt
from compiler import compileFile, visitor
import profile
def main():
VERBOSE = 0
DISPLAY = 0
PROFILE = 0
CONTINUE = 0
opts, args = getopt.getopt(sys.argv[1:], 'vqdcp')
for k, v in opts:
if k == '-v':
VERBOSE = 1
visitor.ASTVisitor.VERBOSE = visitor.ASTVisitor.VERBOSE + 1
if k == '-q':
if sys.platform[:3]=="win":
f = open('nul', 'wb') # /dev/null fails on Windows...
else:
f = open('/dev/null', 'wb')
sys.stdout = f
if k == '-d':
DISPLAY = 1
if k == '-c':
CONTINUE = 1
if k == '-p':
PROFILE = 1
if not args:
print "no files to compile"
else:
for filename in args:
if VERBOSE:
print filename
try:
if PROFILE:
profile.run('compileFile(%r, %r)' % (filename, DISPLAY),
filename + ".prof")
else:
compileFile(filename, DISPLAY)
except SyntaxError as err:
print err
if err.lineno is not None:
print err.lineno
if not CONTINUE:
sys.exit(-1)
if __name__ == "__main__":
main()

View file

@ -1,38 +0,0 @@
#! /usr/bin/env python
"""Print names of all methods defined in module
This script demonstrates use of the visitor interface of the compiler
package.
"""
import compiler
class MethodFinder:
"""Print the names of all the methods
Each visit method takes two arguments, the node and its current
scope. The scope is the name of the current class or None.
"""
def visitClass(self, node, scope=None):
self.visit(node.code, node.name)
def visitFunction(self, node, scope=None):
if scope is not None:
print "%s.%s" % (scope, node.name)
self.visit(node.code, None)
def main(files):
mf = MethodFinder()
for file in files:
f = open(file)
buf = f.read()
f.close()
ast = compiler.parse(buf)
compiler.walk(ast, mf)
if __name__ == "__main__":
import sys
main(sys.argv[1:])

View file

@ -1,47 +0,0 @@
#! /usr/bin/env python
import marshal
import os
import dis
import types
def dump(obj):
print obj
for attr in dir(obj):
if attr.startswith('co_'):
val = getattr(obj, attr)
print "\t", attr, repr(val)
def loadCode(path):
f = open(path)
f.read(8)
co = marshal.load(f)
f.close()
return co
def walk(co, match=None):
if match is None or co.co_name == match:
dump(co)
print
dis.dis(co)
for obj in co.co_consts:
if type(obj) == types.CodeType:
walk(obj, match)
def load(filename, codename=None):
co = loadCode(filename)
walk(co, codename)
if __name__ == "__main__":
import sys
if len(sys.argv) == 3:
filename, codename = sys.argv[1:]
else:
filename = sys.argv[1]
codename = None
if filename.endswith('.py'):
buf = open(filename).read()
co = compile(buf, filename, "exec")
walk(co)
else:
load(filename, codename)

View file

@ -1,83 +0,0 @@
"""Run the Python regression test using the compiler
This test runs the standard Python test suite using bytecode generated
by this compiler instead of by the builtin compiler.
The regression test is run with the interpreter in verbose mode so
that import problems can be observed easily.
"""
from compiler import compileFile
import os
import sys
import test
import tempfile
def copy_test_suite():
dest = tempfile.mkdtemp()
os.system("cp -r %s/* %s" % (test.__path__[0], dest))
print "Creating copy of test suite in", dest
return dest
def copy_library():
dest = tempfile.mkdtemp()
libdir = os.path.split(test.__path__[0])[0]
print "Found standard library in", libdir
print "Creating copy of standard library in", dest
os.system("cp -r %s/* %s" % (libdir, dest))
return dest
def compile_files(dir):
print "Compiling", dir, "\n\t",
line_len = 10
for file in os.listdir(dir):
base, ext = os.path.splitext(file)
if ext == '.py':
source = os.path.join(dir, file)
line_len = line_len + len(file) + 1
if line_len > 75:
print "\n\t",
line_len = len(source) + 9
print file,
try:
compileFile(source)
except SyntaxError as err:
print err
continue
# make sure the .pyc file is not over-written
os.chmod(source + "c", 444)
elif file == 'CVS':
pass
else:
path = os.path.join(dir, file)
if os.path.isdir(path):
print
print
compile_files(path)
print "\t",
line_len = 10
print
def run_regrtest(lib_dir):
test_dir = os.path.join(lib_dir, "test")
os.chdir(test_dir)
os.system("PYTHONPATH=%s %s -v regrtest.py" % (lib_dir, sys.executable))
def cleanup(dir):
os.system("rm -rf %s" % dir)
def raw_input(prompt):
sys.stdout.write(prompt)
sys.stdout.flush()
return sys.stdin.readline()
def main():
lib_dir = copy_library()
compile_files(lib_dir)
run_regrtest(lib_dir)
raw_input("Cleanup?")
cleanup(lib_dir)
if __name__ == "__main__":
main()

View file

@ -1,43 +0,0 @@
import compiler
import dis
import types
def extract_code_objects(co):
l = [co]
for const in co.co_consts:
if type(const) == types.CodeType:
l.append(const)
return l
def compare(a, b):
if not (a.co_name == "?" or a.co_name.startswith('<lambda')):
assert a.co_name == b.co_name, (a, b)
if a.co_stacksize != b.co_stacksize:
print "stack mismatch %s: %d vs. %d" % (a.co_name,
a.co_stacksize,
b.co_stacksize)
if a.co_stacksize > b.co_stacksize:
print "good code"
dis.dis(a)
print "bad code"
dis.dis(b)
assert 0
def main(files):
for file in files:
print file
buf = open(file).read()
try:
co1 = compile(buf, file, "exec")
except SyntaxError:
print "skipped"
continue
co2 = compiler.compile(buf, file, "exec")
co1l = extract_code_objects(co1)
co2l = extract_code_objects(co2)
for a, b in zip(co1l, co2l):
compare(a, b)
if __name__ == "__main__":
import sys
main(sys.argv[1:])