mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-01 07:33:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Update Demo/parser directory; backport unparse fixes from py3k.

Browse source
This commit is contained in:
Mark Dickinson 2010-06-30 14:19:56 +00:00
parent 4aa5f6f1f2
commit 2037913507
5 changed files with 297 additions and 170 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Demo/parser/README
View file

@ -6,26 +6,27 @@ It also contains examples for the AST parser.
Files:
------
FILES -- list of files associated with the parser module.
FILES -- list of files associated with the parser module.
README -- this file.
README -- this file.
example.py -- module that uses the `parser' module to extract
information from the parse tree of Python source
code.
docstring.py -- sample source file containing only a module docstring.
docstring.py -- sample source file containing only a module docstring.
example.py -- module that uses the `parser' module to extract
information from the parse tree of Python source
code.
simple.py -- sample source containing a "short form" definition.
simple.py -- sample source containing a "short form" definition.
source.py -- sample source code used to demonstrate ability to
handle nested constructs easily using the functions
and classes in example.py.
source.py -- sample source code used to demonstrate ability to
handle nested constructs easily using the functions
and classes in example.py.
test_parser.py program to put the parser module through its paces.
test_parser.py program to put the parser module through its paces.
unparse.py AST (2.5) based example to recreate source code
from an AST. This is incomplete; contributions
are welcome.
test_unparse.py tests for the unparse module
unparse.py AST (2.7) based example to recreate source code
from an AST.
Enjoy!

12
Demo/parser/test_parser.py
View file

@ -11,19 +11,19 @@ def testChunk(t, fileName):
global _numFailed
print '----', fileName,
try:
ast = parser.suite(t)
tup = parser.ast2tuple(ast)
# this discards the first AST; a huge memory savings when running
st = parser.suite(t)
tup = parser.st2tuple(st)
# this discards the first ST; a huge memory savings when running
# against a large source file like Tkinter.py.
ast = None
new = parser.tuple2ast(tup)
st = None
new = parser.tuple2st(tup)
except parser.ParserError, err:
print
print 'parser module raised exception on input file', fileName + ':'
traceback.print_exc()
_numFailed = _numFailed + 1
else:
if tup != parser.ast2tuple(new):
if tup != parser.st2tuple(new):
print
print 'parser module failed on input file', fileName
_numFailed = _numFailed + 1

172
Demo/parser/test_unparse.py
View file

@ -1,12 +1,20 @@
import unittest
from test import test_support
import cStringIO
import sys
import os
import tokenize
import ast
import _ast
import unparse
forelse = """\
def read_pyfile(filename):
"""Read and return the contents of a Python source file (as a
string), taking into account the file encoding."""
with open(filename, "r") as pyfile:
source = pyfile.read()
return source
for_else = """\
def f():
for x in range(10):
break
@ -15,7 +23,7 @@ def f():
z = 3
"""
whileelse = """\
while_else = """\
def g():
while True:
break
@ -24,16 +32,63 @@ def g():
z = 3
"""
class UnparseTestCase(unittest.TestCase):
# Tests for specific bugs found in earlier versions of unparse
relative_import = """\
from . import fred
from .. import barney
from .australia import shrimp as prawns
"""
class_decorator = """\
@f1(arg)
@f2
class Foo: pass
"""
elif1 = """\
if cond1:
suite1
elif cond2:
suite2
else:
suite3
"""
elif2 = """\
if cond1:
suite1
elif cond2:
suite2
"""
try_except_finally = """\
try:
suite1
except ex1:
suite2
except ex2:
suite3
else:
suite4
finally:
suite5
"""
class ASTTestCase(unittest.TestCase):
def assertASTEqual(self, ast1, ast2):
dump1 = ast.dump(ast1)
dump2 = ast.dump(ast2)
self.assertEqual(ast.dump(ast1), ast.dump(ast2))
def check_roundtrip(self, code1, filename="internal"):
ast1 = compile(code1, filename, "exec", _ast.PyCF_ONLY_AST)
ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST)
unparse_buffer = cStringIO.StringIO()
unparse.Unparser(ast1, unparse_buffer)
code2 = unparse_buffer.getvalue()
ast2 = compile(code2, filename, "exec", _ast.PyCF_ONLY_AST)
self.assertEqual(ast.dump(ast1), ast.dump(ast2))
ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST)
self.assertASTEqual(ast1, ast2)
class UnparseTestCase(ASTTestCase):
# Tests for specific bugs found in earlier versions of unparse
def test_del_statement(self):
self.check_roundtrip("del x, y, z")
@ -43,23 +98,116 @@ class UnparseTestCase(unittest.TestCase):
self.check_roundtrip("13 >> 7")
def test_for_else(self):
self.check_roundtrip(forelse)
self.check_roundtrip(for_else)
def test_while_else(self):
self.check_roundtrip(whileelse)
self.check_roundtrip(while_else)
def test_unary_parens(self):
self.check_roundtrip("(-1)**7")
self.check_roundtrip("(-1.)**8")
self.check_roundtrip("(-1j)**6")
self.check_roundtrip("not True or False")
self.check_roundtrip("True or not False")
def test_integer_parens(self):
self.check_roundtrip("3 .__abs__()")
def test_huge_float(self):
self.check_roundtrip("1e1000")
self.check_roundtrip("-1e1000")
self.check_roundtrip("1e1000j")
self.check_roundtrip("-1e1000j")
def test_min_int(self):
self.check_roundtrip(str(-sys.maxint-1))
self.check_roundtrip("-(%s)" % (sys.maxint + 1))
def test_imaginary_literals(self):
self.check_roundtrip("7j")
self.check_roundtrip("-7j")
self.check_roundtrip("-(7j)")
self.check_roundtrip("0j")
self.check_roundtrip("-0j")
self.check_roundtrip("-(0j)")
def test_negative_zero(self):
self.check_roundtrip("-0")
self.check_roundtrip("-(0)")
self.check_roundtrip("-0b0")
self.check_roundtrip("-(0b0)")
self.check_roundtrip("-0o0")
self.check_roundtrip("-(0o0)")
self.check_roundtrip("-0x0")
self.check_roundtrip("-(0x0)")
def test_lambda_parentheses(self):
self.check_roundtrip("(lambda: int)()")
def test_chained_comparisons(self):
self.check_roundtrip("1 < 4 <= 5")
self.check_roundtrip("a is b is c is not d")
def test_function_arguments(self):
self.check_roundtrip("def f(): pass")
self.check_roundtrip("def f(a): pass")
self.check_roundtrip("def f(b = 2): pass")
self.check_roundtrip("def f(a, b): pass")
self.check_roundtrip("def f(a, b = 2): pass")
self.check_roundtrip("def f(a = 5, b = 2): pass")
self.check_roundtrip("def f(*args, **kwargs): pass")
def test_relative_import(self):
self.check_roundtrip(relative_import)
def test_bytes(self):
self.check_roundtrip("b'123'")
def test_set_literal(self):
self.check_roundtrip("{'a', 'b', 'c'}")
def test_set_comprehension(self):
self.check_roundtrip("{x for x in range(5)}")
def test_dict_comprehension(self):
self.check_roundtrip("{x: x*x for x in range(10)}")
def test_class_decorators(self):
self.check_roundtrip(class_decorator)
def test_elifs(self):
self.check_roundtrip(elif1)
self.check_roundtrip(elif2)
def test_try_except_finally(self):
self.check_roundtrip(try_except_finally)
class DirectoryTestCase(ASTTestCase):
"""Test roundtrip behaviour on all files in Lib and Lib/test."""
# test directories, relative to the root of the distribution
test_directories = 'Lib', os.path.join('Lib', 'test')
def test_files(self):
# get names of files to test
dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
names = []
for d in self.test_directories:
test_dir = os.path.join(dist_dir, d)
for n in os.listdir(test_dir):
if n.endswith('.py') and not n.startswith('bad'):
names.append(os.path.join(test_dir, n))
for filename in names:
if test_support.verbose:
print('Testing %s' % filename)
source = read_pyfile(filename)
self.check_roundtrip(source)
def test_main():
test_support.run_unittest(UnparseTestCase)
test_support.run_unittest(UnparseTestCase, DirectoryTestCase)
if __name__ == '__main__':
test_main()

100
Demo/parser/texipre.dat
View file

@ -1,100 +0,0 @@
\input texinfo @c -*-texinfo-*-
@c %**start of header
@setfilename parser.info
@settitle Python Parser Module Reference
@setchapternewpage odd
@footnotestyle end
@c %**end of header
@ifinfo
This file describes the interfaces
published by the optional @code{parser} module and gives examples of
how they may be used. It contains the same text as the chapter on the
@code{parser} module in the @cite{Python Library Reference}, but is
presented as a separate document.
Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
Virginia Polytechnic Institute and State University, Blacksburg,
Virginia, USA. Portions of the software copyright 1991-1995 by
Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is
permitted under the terms associated with the main Python distribution,
with the additional restriction that this additional notice be included
and maintained on all distributed copies.
All Rights Reserved
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the names of Fred L. Drake, Jr. and
Virginia Polytechnic Institute and State University not be used in
advertising or publicity pertaining to distribution of the software
without specific, written prior permission.
FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
@end ifinfo
@titlepage
@title Python Parser Module Reference
@author Fred L. Drake, Jr.
@c The following two commands start the copyright page.
@page
@vskip 0pt plus 1filll
Copyright 1995-1996 by Fred L. Drake, Jr., Reston, Virginia, USA, and
Virginia Polytechnic Institute and State University, Blacksburg,
Virginia, USA. Portions of the software copyright 1991-1995 by
Stichting Mathematisch Centrum, Amsterdam, The Netherlands. Copying is
permitted under the terms associated with the main Python distribution,
with the additional restriction that this additional notice be included
and maintained on all distributed copies.
@center All Rights Reserved
Permission to use, copy, modify, and distribute this software and its
documentation for any purpose and without fee is hereby granted,
provided that the above copyright notice appear in all copies and that
both that copyright notice and this permission notice appear in
supporting documentation, and that the names of Fred L. Drake, Jr. and
Virginia Polytechnic Institute and State University not be used in
advertising or publicity pertaining to distribution of the software
without specific, written prior permission.
FRED L. DRAKE, JR. AND VIRGINIA POLYTECHNIC INSTITUTE AND STATE
UNIVERSITY DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
EVENT SHALL FRED L. DRAKE, JR. OR VIRGINIA POLYTECHNIC INSTITUTE AND
STATE UNIVERSITY BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL
DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
PERFORMANCE OF THIS SOFTWARE.
@end titlepage
@node Top, Overview, (dir), (dir)
@top The Python Parser Module
@ifinfo
This file describes the interfaces
published by the optional @code{parser} module and gives examples of
how they may be used. It contains the same text as the chapter on the
@code{parser} module in the @cite{Python Library Reference}, but is
presented as a separate document.
This version corresponds to Python version 1.4 (1 Sept. 1996).
@end ifinfo
@c placeholder for the master menu -- patched by texinfo-all-menus-update
@menu
@end menu

154
Demo/parser/unparse.py
View file

@ -1,15 +1,19 @@
"Usage: unparse.py <path to source file>"
import sys
import _ast
import ast
import cStringIO
import os
# Large float and imaginary literals get turned into infinities in the AST.
# We unparse those infinities to INFSTR.
INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
def interleave(inter, f, seq):
"""Call f on each item in seq, calling inter() in between.
"""
seq = iter(seq)
try:
f(seq.next())
f(next(seq))
except StopIteration:
pass
else:
@ -20,15 +24,16 @@ def interleave(inter, f, seq):
class Unparser:
"""Methods in this class recursively traverse an AST and
output source code for the abstract syntax; original formatting
is disregarged. """
is disregarded. """
def __init__(self, tree, file = sys.stdout):
"""Unparser(tree, file=sys.stdout) -> None.
Print the source for tree to file."""
self.f = file
self.future_imports = []
self._indent = 0
self.dispatch(tree)
print >>self.f,""
self.f.write("")
self.f.flush()
def fill(self, text = ""):
@ -79,11 +84,16 @@ class Unparser:
interleave(lambda: self.write(", "), self.dispatch, t.names)
def _ImportFrom(self, t):
# A from __future__ import may affect unparsing, so record it.
if t.module and t.module == '__future__':
self.future_imports.extend(n.name for n in t.names)
self.fill("from ")
self.write(t.module)
self.write("." * t.level)
if t.module:
self.write(t.module)
self.write(" import ")
interleave(lambda: self.write(", "), self.dispatch, t.names)
# XXX(jpe) what is level for?
def _Assign(self, t):
self.fill()
@ -186,10 +196,14 @@ class Unparser:
self.leave()
def _TryFinally(self, t):
self.fill("try")
self.enter()
self.dispatch(t.body)
self.leave()
if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
# try-except-finally
self.dispatch(t.body)
else:
self.fill("try")
self.enter()
self.dispatch(t.body)
self.leave()
self.fill("finally")
self.enter()
@ -202,7 +216,7 @@ class Unparser:
self.write(" ")
self.dispatch(t.type)
if t.name:
self.write(", ")
self.write(" as ")
self.dispatch(t.name)
self.enter()
self.dispatch(t.body)
@ -210,6 +224,9 @@ class Unparser:
def _ClassDef(self, t):
self.write("\n")
for deco in t.decorator_list:
self.fill("@")
self.dispatch(deco)
self.fill("class "+t.name)
if t.bases:
self.write("(")
@ -251,9 +268,18 @@ class Unparser:
self.fill("if ")
self.dispatch(t.test)
self.enter()
# XXX elif?
self.dispatch(t.body)
self.leave()
# collapse nested ifs into equivalent elifs.
while (t.orelse and len(t.orelse) == 1 and
isinstance(t.orelse[0], ast.If)):
t = t.orelse[0]
self.fill("elif ")
self.dispatch(t.test)
self.enter()
self.dispatch(t.body)
self.leave()
# final else
if t.orelse:
self.fill("else")
self.enter()
@ -284,7 +310,17 @@ class Unparser:
# expr
def _Str(self, tree):
self.write(repr(tree.s))
# if from __future__ import unicode_literals is in effect,
# then we want to output string literals using a 'b' prefix
# and unicode literals with no prefix.
if "unicode_literals" not in self.future_imports:
self.write(repr(tree.s))
elif isinstance(tree.s, str):
self.write("b" + repr(tree.s))
elif isinstance(tree.s, unicode):
self.write(repr(tree.s).lstrip("u"))
else:
assert False, "shouldn't get here"
def _Name(self, t):
self.write(t.id)
@ -295,16 +331,14 @@ class Unparser:
self.write("`")
def _Num(self, t):
# There are no negative numeric literals in Python; however,
# some optimizations produce a negative Num in the AST. Add
# parentheses to avoid turning (-1)**2 into -1**2.
strnum = repr(t.n)
if strnum.startswith("-"):
repr_n = repr(t.n)
# Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
if repr_n.startswith("-"):
self.write("(")
self.write(strnum)
# Substitute overflowing decimal literal for AST infinities.
self.write(repr_n.replace("inf", INFSTR))
if repr_n.startswith("-"):
self.write(")")
else:
self.write(repr(t.n))
def _List(self, t):
self.write("[")
@ -325,6 +359,22 @@ class Unparser:
self.dispatch(gen)
self.write(")")
def _SetComp(self, t):
self.write("{")
self.dispatch(t.elt)
for gen in t.generators:
self.dispatch(gen)
self.write("}")
def _DictComp(self, t):
self.write("{")
self.dispatch(t.key)
self.write(": ")
self.dispatch(t.value)
for gen in t.generators:
self.dispatch(gen)
self.write("}")
def _comprehension(self, t):
self.write(" for ")
self.dispatch(t.target)
@ -343,13 +393,20 @@ class Unparser:
self.dispatch(t.orelse)
self.write(")")
def _Set(self, t):
assert(t.elts) # should be at least one element
self.write("{")
interleave(lambda: self.write(", "), self.dispatch, t.elts)
self.write("}")
def _Dict(self, t):
self.write("{")
def writem((k, v)):
def write_pair(pair):
(k, v) = pair
self.dispatch(k)
self.write(": ")
self.dispatch(v)
interleave(lambda: self.write(", "), writem, zip(t.keys, t.values))
interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
self.write("}")
def _Tuple(self, t):
@ -367,7 +424,17 @@ class Unparser:
self.write("(")
self.write(self.unop[t.op.__class__.__name__])
self.write(" ")
self.dispatch(t.operand)
# If we're applying unary minus to a number, parenthesize the number.
# This is necessary: -2147483648 is different from -(2147483648) on
# a 32-bit machine (the first is an int, the second a long), and
# -7j is different from -(7j). (The first has real part 0.0, the second
# has real part -0.0.)
if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
self.write("(")
self.dispatch(t.operand)
self.write(")")
else:
self.dispatch(t.operand)
self.write(")")
binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
@ -390,7 +457,7 @@ class Unparser:
self.dispatch(e)
self.write(")")
boolops = {_ast.And: 'and', _ast.Or: 'or'}
boolops = {ast.And: 'and', ast.Or: 'or'}
def _BoolOp(self, t):
self.write("(")
s = " %s " % self.boolops[t.op.__class__]
@ -399,6 +466,11 @@ class Unparser:
def _Attribute(self,t):
self.dispatch(t.value)
# Special case: 3.__abs__() is a syntax error, so if t.value
# is an integer literal then we need to either parenthesize
# it or add an extra space to get 3 .__abs__().
if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
self.write(" ")
self.write(".")
self.write(t.attr)
@ -455,21 +527,24 @@ class Unparser:
# others
def _arguments(self, t):
first = True
nonDef = len(t.args)-len(t.defaults)
for a in t.args[0:nonDef]:
if first:first = False
else: self.write(", ")
self.dispatch(a)
for a,d in zip(t.args[nonDef:], t.defaults):
# normal arguments
defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
for a,d in zip(t.args, defaults):
if first:first = False
else: self.write(", ")
self.dispatch(a),
self.write("=")
self.dispatch(d)
if d:
self.write("=")
self.dispatch(d)
# varargs
if t.vararg:
if first:first = False
else: self.write(", ")
self.write("*"+t.vararg)
self.write("*")
self.write(t.vararg)
# kwargs
if t.kwarg:
if first:first = False
else: self.write(", ")
@ -481,10 +556,12 @@ class Unparser:
self.dispatch(t.value)
def _Lambda(self, t):
self.write("(")
self.write("lambda ")
self.dispatch(t.args)
self.write(": ")
self.dispatch(t.body)
self.write(")")
def _alias(self, t):
self.write(t.name)
@ -492,8 +569,9 @@ class Unparser:
self.write(" as "+t.asname)
def roundtrip(filename, output=sys.stdout):
source = open(filename).read()
tree = compile(source, filename, "exec", _ast.PyCF_ONLY_AST)
with open(filename, "r") as pyfile:
source = pyfile.read()
tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
Unparser(tree, output)
@ -502,7 +580,7 @@ def testdir(a):
try:
names = [n for n in os.listdir(a) if n.endswith('.py')]
except OSError:
print >> sys.stderr, "Directory not readable: %s" % a
sys.stderr.write("Directory not readable: %s" % a)
else:
for n in names:
fullname = os.path.join(a, n)
@ -511,7 +589,7 @@ def testdir(a):
print 'Testing %s' % fullname
try:
roundtrip(fullname, output)
except Exception, e:
except Exception as e:
print ' Failed to compile, exception is %s' % repr(e)
elif os.path.isdir(fullname):
testdir(fullname)