mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-08-26 03:34:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

New regression testing harness.

Browse source 
(testall.out): obsolete.  see individual test output files in the
Output directory.

(testall.py): now contains only the list of all tests

(autotest.py): The new testing harness.  Supports -g option to
generate output files instead of verifying; -w option to warn about
non-tested optional modules; additional command line arguments to
generate or test individual tests (useful for generating output file
of new tests).
This commit is contained in:
Barry Warsaw 1996-12-10 19:51:10 +00:00
parent 0aa9ee65ab
commit 272c00b634
3 changed files with 130 additions and 242 deletions
Download patch file Download diff file Expand all files Collapse all files

146
Lib/test/autotest.py
View file

@ -1,34 +1,58 @@
# Automatic Python regression test. """
# Automatic Python regression test.
# Some essential parts of the Python interpreter are tested by the module
# 'testall'. (Despite its name, it doesn't test everything -- that would The list of individual tests is contained in the `testall' module.
# be a truly Herculean task!) When a test fails, 'testall' raises an These test some (but not all) essential parts of the Python
# exception. When all tests succeed, it produces quite a lot of output. interpreter and built-in modules. When a test fails, an exception is
# raised and testing halts. When a test succeeds, it can produce quite
# For a normal regression test, this output is never looked at unless a lot of output, which is compared against the output from a previous
# something goes wrong. Thus, it would be wise to suppress the output run. If a difference is noticed it raises an exception; if all is
# normally. This module does that, but it doesn't just throw the output well, it prints nothing except 'All tests OK.' at the very end.
# from 'testall' away -- it compares it with the output from a previous
# run. If a difference is noticed it raises an exception; if all is well, The output from a previous run is supposed to be contained in separate
# it prints nothing except 'All tests OK.' at the very end. files (one per test) in the `Output' subdirectory somewhere on the
# search path for modules (sys.path, initialized from $PYTHONPATH plus
# The output from a previous run is supposed to be in a file 'testall.out' some default places).
# somewhere on the search path for modules (sys.path, initialized from
# $PYTHONPATH plus some default places). Of course, if the normal output of the tests is changed because the
# tests have been changed (rather than a test producing the wrong
# Of course, if the normal output of the tests is changed because the output), 'autotest' will fail as well. In this case, run 'autotest'
# tests have been changed (rather than a test producing the wrong output), with the -g option.
# 'autotest' will fail as well. In this case, run 'testall' manually
# and direct its output to 'testall.out'. Usage:
#
# The comparison uses (and demonstrates!) a rather new Python feature: %s [-g] [-w] [-h] [test1 [test2 ...]]
# program output that normally goes to stdout (by 'print' statements
# or by writing directly to sys.stdout) can be redirected to an Options:
# arbitrary class instance as long as it has a 'write' method.
-g, --generate : generate the output files instead of verifying
the results
-w, --warn : warn about un-importable tests
-h, --help : print this message
If individual tests are provided on the command line, only those tests
will be performed or generated. Otherwise, all tests (as contained in
testall.py) will be performed.
"""
import os import os
import sys import sys
import getopt
import traceback
from test_support import *
# Exception raised when the test failed (not the same as in test_support)
TestFailed = 'autotest.TestFailed'
# defaults
generate = 0
warn = 0
# Function to find a file somewhere on sys.path # Function to find a file somewhere on sys.path
def findfile(filename): def findfile(filename):
for dirname in sys.path: for dirname in sys.path:
@ -37,9 +61,8 @@ def findfile(filename):
return fullname return fullname
return filename # Will cause exception later return filename # Will cause exception later
# Exception raised when the test failed (not the same as in test_support)
TestFailed = 'autotest.TestFailed'
# Class substituted for sys.stdout, to compare it with the given file # Class substituted for sys.stdout, to compare it with the given file
class Compare: class Compare:
def __init__(self, filename): def __init__(self, filename):
@ -50,18 +73,73 @@ class Compare:
raise TestFailed, \ raise TestFailed, \
'Writing: '+`data`+', expected: '+`expected` 'Writing: '+`data`+', expected: '+`expected`
def close(self): def close(self):
leftover = self.fp.read()
if leftover:
raise TestFailed, 'Unread: '+`leftover`
self.fp.close() self.fp.close()
# The main program # The main program
def main(): def usage(status):
import sys print __doc__ % sys.argv[0]
filename = findfile('testall.out') sys.exit(status)
def do_one_test(t, outdir):
filename = os.path.join(outdir, t)
real_stdout = sys.stdout real_stdout = sys.stdout
try: try:
sys.stdout = Compare(filename) if generate:
import testall print 'Generating:', filename
sys.stdout = open(filename, 'w')
else:
sys.stdout = Compare(filename)
print t
unload(t)
try:
__import__(t, globals(), locals())
except ImportError, msg:
if warn:
sys.stderr.write(msg+': Un-installed'
' optional module?\n')
finally: finally:
sys.stdout.close()
sys.stdout = real_stdout sys.stdout = real_stdout
def main():
global generate
global warn
try:
opts, args = getopt.getopt(
sys.argv[1:], 'ghw',
['generate', 'help', 'warn'])
except getopt.error, msg:
print msg
usage(1)
for opt, val in opts:
if opt in ['-h', '--help']:
usage(0)
elif opt in ['-g', '--generate']:
generate = 1
elif opt in ['-w', '--warn']:
warn = 1
# find the output directory
outdir = findfile('Output')
if args:
tests = args
else:
import testall
tests = testall.tests
for test in tests:
try:
do_one_test(test, outdir)
except TestFailed, msg:
print 'Failure of test:', test
traceback.print_exc()
print 'All tests OK.' print 'All tests OK.'
main() main()

184
Lib/test/testall.out
View file

@ -1,184 +0,0 @@
test_grammar
1. Parser
1.1 Tokens
1.1.1 Backslashes
1.1.2 Numeric literals
1.1.2.1 Plain integers
1.1.2.2 Long integers
1.1.2.3 Floating point
1.1.3 String literals
1.2 Grammar
single_input
file_input
expr_input
eval_input
funcdef
simple_stmt
expr_stmt
print_stmt
1 2 3
1 2 3
1 1 1
del_stmt
pass_stmt
flow_stmt
break_stmt
continue_stmt
return_stmt
raise_stmt
import_stmt
global_stmt
exec_stmt
if_stmt
while_stmt
for_stmt
try_stmt
suite
test
comparison
binary mask ops
shift ops
additive ops
multiplicative ops
unary ops
selectors
atoms
classdef
test_opcodes
2. Opcodes
XXX Not yet fully implemented
2.1 try inside for loop
2.2 raise class exceptions
test_operations
3. Operations
XXX Not yet implemented
test_builtin
4. Built-in functions
test_b1
__import__
abs
apply
callable
chr
cmp
coerce
compile
delattr
dir
divmod
eval
execfile
filter
float
getattr
hasattr
hash
hex
id
int
len
long
map
max
min
test_b2
oct
open
ord
pow
range
input and raw_input
testing
testing
reduce
reload
repr
round
setattr
str
tuple
type
vars
xrange
test_exceptions
5. Built-in exceptions
AttributeError
EOFError
IOError
ImportError
IndexError
KeyError
KeyboardInterrupt
(not testable in a script)
MemoryError
(not safe to test)
NameError
OverflowError
RuntimeError
(not used any more?)
SyntaxError
SystemError
(hard to reproduce)
SystemExit
TypeError
ValueError
ZeroDivisionError
test_types
6. Built-in types
6.1 Truth value testing
6.2 Boolean operations
6.3 Comparisons
6.4 Numeric types (mostly conversions)
6.4.1 32-bit integers
6.4.2 Long integers
6.4.3 Floating point numbers
6.5 Sequence types
6.5.1 Strings
6.5.2 Tuples
6.5.3 Lists
6.5.3a Additional list operations
6.6 Mappings == Dictionaries
test_math
math module, testing with eps 1e-05
constants
acos
asin
atan
atan2
ceil
cos
cosh
exp
fabs
floor
fmod
frexp
hypot
ldexp
log
log10
modf
pow
sin
sinh
sqrt
tan
tanh
test_time
test_array
test_strop
test_md5
MD5 test suite:
MD5 ("") = d41d8cd98f00b204e9800998ecf8427e
MD5 ("a") = 0cc175b9c0f1b6a831c399e269772661
MD5 ("abc") = 900150983cd24fb0d6963f7d28e17f72
MD5 ("message digest") = f96b697d7cb7938d525a2f31aaf161d0
MD5 ("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
MD5 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
MD5 ("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
test_cmath
test_crypt
Test encryption: abeTm2pJKypwA
test_dbm
test_new
Passed all tests.

42
Lib/test/testall.py
View file

@ -1,25 +1,19 @@
# testall.py -- a regression test for the Python interpreter. # This file now contains only the list of separate regression tests.
# To run the tests, execute "import testall" in a clean interpreter. # All of the testing harness is now contained in autotest.py.
# It is a good idea to do this whenever you build a new interpreter.
# Remember to add new tests when new features are added!
import sys tests = ['test_grammar',
from test_support import * 'test_opcodes',
'test_operations',
print 'test_grammar' 'test_builtin',
forget('test_grammar') 'test_exceptions',
import test_grammar 'test_types',
'test_math',
for t in ['test_opcodes', 'test_operations', 'test_builtin', 'test_time',
'test_exceptions', 'test_types', 'test_math', 'test_time', 'test_array',
'test_array', 'test_strop', 'test_md5', 'test_cmath', 'test_strop',
'test_crypt', 'test_dbm', 'test_new', 'test_md5',
]: 'test_cmath',
print t 'test_crypt',
unload(t) 'test_dbm',
try: 'test_new',
__import__(t, globals(), locals()) ]
except ImportError, msg:
sys.stderr.write('%s. Uninstalled optional module?\n' % msg)
print 'Passed all tests.'