Add -t option to allow easy test selection.

Action verbose option correctly.
Tweak operation counts. Add empty and new instances tests.
Enable comparisons across different warp factors. Change version.

Tools/pybench/Arithmetic.py

@@ -4,7 +4,7 @@ class SimpleIntegerArithmetic(Test):
 
     version = 0.3
     operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
-    rounds = 120000
+    rounds = 1200*21
 
     def test(self):
 
@@ -159,7 +159,7 @@ class SimpleFloatArithmetic(Test):
 
     version = 0.3
     operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
-    rounds = 100000
+    rounds = 1000*30
 
     def test(self):
 
@@ -314,7 +314,7 @@ class SimpleIntFloatArithmetic(Test):
 
     version = 0.3
     operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
-    rounds = 120000
+    rounds = 1200*30
 
     def test(self):
 
@@ -470,7 +470,7 @@ class SimpleLongArithmetic(Test):
 
     version = 0.3
     operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
-    rounds = 30000
+    rounds = 300*32
 
     def test(self):
 
@@ -625,7 +625,7 @@ class SimpleComplexArithmetic(Test):
 
     version = 0.3
     operations = 5 * (3 + 5 + 5 + 3 + 3 + 3)
-    rounds = 40000
+    rounds = 400*27
 
     def test(self):
 

Tools/pybench/Calls.py

@@ -4,7 +4,7 @@ class PythonFunctionCalls(Test):
 
     version = 0.3
     operations = 5*(1+4+4+2)
-    rounds = 60000
+    rounds = 600*22
 
     def test(self):
 
@@ -113,7 +113,7 @@ class BuiltinFunctionCalls(Test):
 
     version = 0.4
     operations = 5*(2+5+5+5)
-    rounds = 30000
+    rounds = 300*24
 
     def test(self):
 
@@ -234,7 +234,7 @@ class PythonMethodCalls(Test):
 
     version = 0.3
     operations = 5*(6 + 5 + 4)
-    rounds = 20000
+    rounds = 200*27
 
     def test(self):
 
@@ -376,7 +376,7 @@ class Recursion(Test):
 
     version = 0.3
     operations = 5
-    rounds = 50000
+    rounds = 500*21
 
     def test(self):
 

Tools/pybench/Constructs.py

@@ -4,7 +4,7 @@ class IfThenElse(Test):
 
     version = 0.31
     operations = 30*3 # hard to say...
-    rounds = 150000
+    rounds = 1500*27
 
     def test(self):
 
@@ -471,7 +471,7 @@ class NestedForLoops(Test):
 
     version = 0.3
     operations = 1000*10*5
-    rounds = 150
+    rounds = 100
 
     def test(self):
 
@@ -496,7 +496,7 @@ class ForLoops(Test):
 
     version = 0.1
     operations = 5 * 5
-    rounds = 8000
+    rounds = 80*25
 
     def test(self):
 

Tools/pybench/Dict.py

@@ -4,7 +4,7 @@ class DictCreation(Test):
 
     version = 0.3
     operations = 5*(5 + 5)
-    rounds = 60000
+    rounds = 600*24
 
     def test(self):
 
@@ -79,7 +79,7 @@ class DictWithStringKeys(Test):
 
     version = 0.1
     operations = 5*(6 + 6)
-    rounds = 200000
+    rounds = 2000*30
 
     def test(self):
 
@@ -168,7 +168,7 @@ class DictWithFloatKeys(Test):
 
     version = 0.1
     operations = 5*(6 + 6)
-    rounds = 200000
+    rounds = 20000
 
     def test(self):
 
@@ -257,7 +257,7 @@ class DictWithIntegerKeys(Test):
 
     version = 0.1
     operations = 5*(6 + 6)
-    rounds = 200000
+    rounds = 2000*19
 
     def test(self):
 
@@ -346,7 +346,7 @@ class SimpleDictManipulation(Test):
 
     version = 0.3
     operations = 5*(6 + 6 + 6 + 6)
-    rounds = 50000
+    rounds = 500*44
 
     def test(self):
 

Tools/pybench/Empty.py

@@ -0,0 +1,23 @@
+from pybench import Test
+
+class EmptyTest(Test):
+    """This is just here as a potential measure of repeatability."""
+
+    version = 0.3
+    operations = 1
+    rounds = 60000
+
+    def test(self):
+
+        l = []
+        for i in xrange(self.rounds):
+            pass
+
+
+    def calibrate(self):
+
+        l = []
+
+        for i in xrange(self.rounds):
+            pass
+

Tools/pybench/Exceptions.py

@@ -4,7 +4,7 @@ class TryRaiseExcept(Test):
 
     version = 0.1
     operations = 2 + 3
-    rounds = 60000
+    rounds = 600*25
 
     def test(self):
 
@@ -44,7 +44,7 @@ class TryExcept(Test):
 
     version = 0.1
     operations = 15 * 10
-    rounds = 200000
+    rounds = 2000*16
 
     def test(self):
 

Tools/pybench/Imports.py

@@ -8,7 +8,7 @@ class SecondImport(Test):
 
     version = 0.1
     operations = 5 * 5
-    rounds = 20000
+    rounds = 2000*15
 
     def test(self):
 
@@ -53,7 +53,7 @@ class SecondPackageImport(Test):
 
     version = 0.1
     operations = 5 * 5
-    rounds = 20000
+    rounds = 200*20
 
     def test(self):
 
@@ -97,7 +97,7 @@ class SecondSubmoduleImport(Test):
 
     version = 0.1
     operations = 5 * 5
-    rounds = 20000
+    rounds = 200*17
 
     def test(self):
 

Tools/pybench/Instances.py

@@ -4,7 +4,7 @@ class CreateInstances(Test):
 
     version = 0.2
     operations = 3 + 7 + 4
-    rounds = 60000
+    rounds = 600*17
 
     def test(self):
 

Tools/pybench/Lists.py

@@ -4,7 +4,7 @@ class SimpleListManipulation(Test):
 
     version = 0.3
     operations = 5* (6 + 6 + 6)
-    rounds = 60000
+    rounds = 600*45
 
     def test(self):
 
@@ -132,7 +132,7 @@ class ListSlicing(Test):
 
     version = 0.4
     operations = 25*(3+1+2+1)
-    rounds = 400
+    rounds = 4*45
 
     def test(self):
 
@@ -169,7 +169,7 @@ class SmallLists(Test):
 
     version = 0.3
     operations = 5*(1+ 6 + 6 + 3 + 1)
-    rounds = 60000
+    rounds = 600*15
 
     def test(self):
 

Tools/pybench/Lookups.py

@@ -4,7 +4,7 @@ class SpecialClassAttribute(Test):
 
     version = 0.3
     operations = 5*(12 + 12)
-    rounds = 100000
+    rounds = 1000*16
 
     def test(self):
 
@@ -185,7 +185,7 @@ class NormalClassAttribute(Test):
 
     version = 0.3
     operations = 5*(12 + 12)
-    rounds = 100000
+    rounds = 1000*20
 
     def test(self):
 
@@ -371,7 +371,7 @@ class SpecialInstanceAttribute(Test):
 
     version = 0.3
     operations = 5*(12 + 12)
-    rounds = 100000
+    rounds = 1000*14
 
     def test(self):
 
@@ -559,7 +559,7 @@ class NormalInstanceAttribute(Test):
 
     version = 0.3
     operations = 5*(12 + 12)
-    rounds = 100000
+    rounds = 1000*22
 
     def test(self):
 
@@ -747,7 +747,7 @@ class BuiltinMethodLookup(Test):
 
     version = 0.3
     operations = 5*(3*5 + 3*5)
-    rounds = 70000
+    rounds = 700*15
 
     def test(self):
 

Tools/pybench/NewInstances.py

@@ -0,0 +1,66 @@
+from pybench import Test
+
+class CreateNewInstances(Test):
+
+    version = 0.1
+    operations = 3 + 7 + 4
+    rounds = 60000
+
+    def test(self):
+
+        class c(object):
+            pass
+
+        class d(object):
+            def __init__(self,a,b,c):
+                self.a = a
+                self.b = b
+                self.c = c
+
+        class e(object):
+            def __init__(self,a,b,c=4):
+                self.a = a
+                self.b = b
+                self.c = c
+                self.d = a
+                self.e = b
+                self.f = c
+
+        for i in xrange(self.rounds):
+            o = c()
+            o1 = c()
+            o2 = c()
+            p = d(i,i,3)
+            p1 = d(i,i,3)
+            p2 = d(i,3,3)
+            p3 = d(3,i,3)
+            p4 = d(i,i,i)
+            p5 = d(3,i,3)
+            p6 = d(i,i,i)
+            q = e(i,i,3)
+            q1 = e(i,i,3)
+            q2 = e(i,i,3)
+            q3 = e(i,i)
+
+    def calibrate(self):
+
+        class c(object):
+            pass
+
+        class d(object):
+            def __init__(self,a,b,c):
+                self.a = a
+                self.b = b
+                self.c = c
+
+        class e(object):
+            def __init__(self,a,b,c=4):
+                self.a = a
+                self.b = b
+                self.c = c
+                self.d = a
+                self.e = b
+                self.f = c
+
+        for i in xrange(self.rounds):
+            pass

Tools/pybench/Numbers.py

@@ -4,7 +4,7 @@ class CompareIntegers(Test):
 
     version = 0.1
     operations = 30 * 5
-    rounds = 120000
+    rounds = 1200*21
 
     def test(self):
 
@@ -200,7 +200,7 @@ class CompareFloats(Test):
 
     version = 0.1
     operations = 30 * 5
-    rounds = 60000
+    rounds = 600*27
 
     def test(self):
 
@@ -396,7 +396,7 @@ class CompareFloatsIntegers(Test):
 
     version = 0.1
     operations = 30 * 5
-    rounds = 60000
+    rounds = 600*16
 
     def test(self):
 
@@ -592,7 +592,7 @@ class CompareLongs(Test):
 
     version = 0.1
     operations = 30 * 5
-    rounds = 60000
+    rounds = 600*24
 
     def test(self):
 

Tools/pybench/Setup.py

@@ -17,11 +17,16 @@ Number_of_rounds = 10
 Warp_factor = 20
 
 # Import tests
+#from Empty import *
 from Arithmetic import *
 from Calls import *
 from Constructs import *
 from Lookups import *
 from Instances import *
+try:
+    from NewInstances import *
+except:
+    print "Cannot test new-style objects"
 from Lists import *
 from Tuples import *
 from Dict import *

Tools/pybench/Strings.py

@@ -5,7 +5,7 @@ class ConcatStrings(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 60000
+    rounds = 6000
 
     def test(self):
 
@@ -87,7 +87,7 @@ class CompareStrings(Test):
 
     version = 0.2
     operations = 10 * 5
-    rounds = 200000
+    rounds = 2000*22
 
     def test(self):
 
@@ -169,7 +169,7 @@ class CompareInternedStrings(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 200000
+    rounds = 2000*28
 
     def test(self):
 
@@ -251,7 +251,7 @@ class CreateStringsWithConcat(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 80000
+    rounds = 800*32
 
     def test(self):
 
@@ -326,7 +326,7 @@ class StringSlicing(Test):
 
     version = 0.1
     operations = 5 * 7
-    rounds = 100000
+    rounds = 1000*15
 
     def test(self):
 
@@ -389,7 +389,7 @@ if hasattr('', 'lower'):
 
         version = 0.1
         operations = 3 * (5 + 4 + 2 + 1)
-        rounds = 70000
+        rounds = 14000
 
         def test(self):
 
@@ -462,7 +462,7 @@ if hasattr('', 'lower'):
 
         version = 0.1
         operations = 10 * 7
-        rounds = 80000
+        rounds = 800*24
 
         def test(self):
 

Tools/pybench/Tuples.py

@@ -4,7 +4,7 @@ class TupleSlicing(Test):
 
     version = 0.31
     operations = 3 * 25 * 10 * 7
-    rounds = 400
+    rounds = 100
 
     def test(self):
 
@@ -272,7 +272,7 @@ class SmallTuples(Test):
 
     version = 0.3
     operations = 5*(1 + 3 + 6 + 2)
-    rounds = 80000
+    rounds = 800*16
 
     def test(self):
 

Tools/pybench/Unicode.py

@@ -10,7 +10,7 @@ class ConcatUnicode(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 60000
+    rounds = 600*7
 
     def test(self):
 
@@ -92,7 +92,7 @@ class CompareUnicode(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 150000
+    rounds = 1500*17
 
     def test(self):
 
@@ -174,7 +174,7 @@ class CreateUnicodeWithConcat(Test):
 
     version = 0.1
     operations = 10 * 5
-    rounds = 80000
+    rounds = 800*12
 
     def test(self):
 
@@ -249,7 +249,7 @@ class UnicodeSlicing(Test):
 
     version = 0.1
     operations = 5 * 7
-    rounds = 100000
+    rounds = 10000
 
     def test(self):
 
@@ -310,7 +310,7 @@ class UnicodeMappings(Test):
 
     version = 0.1
     operations = 3 * (5 + 4 + 2 + 1)
-    rounds = 10000
+    rounds = 100*15
 
     def test(self):
 
@@ -383,7 +383,7 @@ class UnicodePredicates(Test):
 
     version = 0.1
     operations = 5 * 9
-    rounds = 100000
+    rounds = 1000*25
 
     def test(self):
 
@@ -460,7 +460,7 @@ else:
 
         version = 0.1
         operations = 5 * 8
-        rounds = 100000
+        rounds = 1000*15
 
         def test(self):
 

Tools/pybench/pybench.py

@@ -35,7 +35,7 @@ WITH THE USE OR PERFORMANCE OF THIS SOFTWARE !
 """
 
 # Version number
-__version__ = '1.3'
+__version__ = '1.4'
 
 #
 # NOTE: Use xrange for all test loops unless you want to face
@@ -98,7 +98,7 @@ class Test:
     # Number of rounds to execute per test run. This should be
     # adjusted to a figure that results in a test run-time of between
     # 20-50 seconds.
-    rounds = 100000
+    rounds = 10000
 
     ### Internal variables
 
@@ -115,6 +115,8 @@ class Test:
 
         if warp > 1:
             self.rounds = self.rounds / warp
+            if self.rounds == 0:
+                self.rounds = 1
             self.warp = warp
         self.times = []
         self.overhead = []
@@ -137,6 +139,7 @@ class Test:
         cruns = self.cruns
         # first calibrate
         offset = 0.0
+        if cruns:
             for i in range(cruns):
                 t = clock()
                 calibrate()
@@ -175,13 +178,18 @@ class Test:
 
     def stat(self):
 
-        """ Returns two value: average time per run and average per
-            operation.
+        """ Returns four values:
+        minimum round time
+        average time per round
+        average time per operation
+        average overhead time
 
+        XXX Should this take warp factors into account?
         """
         runs = len(self.times)
         if runs == 0:
             return 0,0
+        mintime = min(self.times)
         totaltime = reduce(operator.add,self.times,0.0)
         avg = totaltime / float(runs)
         op_avg = totaltime / float(runs * self.rounds * self.operations)
@@ -191,7 +199,7 @@ class Test:
         else:
             # use self.last_timing - not too accurate
             ov_avg = self.last_timing[2]
-        return avg,op_avg,ov_avg
+        return mintime, avg, op_avg, ov_avg
 
 ### Load Setup
 
@@ -210,105 +218,132 @@ class Benchmark:
     roundtime = 0               # Average round time
     version = None              # Benchmark version number (see __init__)
                                 # as float x.yy
-    starttime = None            # Benchmark start time
 
     def __init__(self):
 
         self.tests = {}
         self.version = 0.31
 
-    def load_tests(self,setupmod,warp=1):
+    def load_tests(self, setupmod, warp=1, limitnames="", verbose=0):
 
         self.warp = warp
+        if limitnames:
+            limitnames = re.compile(limitnames, re.I)
+        else:
+            limitnames = None
         tests = self.tests
-        print 'Searching for tests...'
+        if verbose:
+            print 'Searching for tests ...',
         setupmod.__dict__.values()
         for c in setupmod.__dict__.values():
-            if hasattr(c,'is_a_test') and c.__name__ != 'Test':
-                tests[c.__name__] = c(warp)
+            if not hasattr(c,'is_a_test'):
+                continue
+            name = c.__name__
+            if  name == 'Test':
+                continue
+            if limitnames is not None and limitnames.search(name) is None:
+                continue
+            tests[name] = c(warp)
         l = tests.keys()
         l.sort()
+        if verbose:
+            print
             for t in l:
                 print '  ', t
+            print len(l), "tests found"
         print
 
-    def run(self):
+    def run(self, verbose):
 
         tests = self.tests.items()
         tests.sort()
         clock = time.clock
-        print 'Running %i round(s) of the suite: ' % self.rounds
+        print 'Running %i round(s) of the suite at warp factor %i:' % (self.rounds, self.warp)
         print
-        self.starttime = time.time()
         roundtime = clock()
         for i in range(self.rounds):
+            roundstarttime = clock()
+            if verbose:
                 print ' Round %-25i  real   abs    overhead' % (i+1)
             for j in range(len(tests)):
                 name,t = tests[j]
+                if verbose:
                     print '%30s:' % name,
                 t.run()
+                if verbose:
                     print '  %.3fr %.3fa %.3fo' % t.last_timing
+            if verbose:
                 print '                                 ----------------------'
                 print '            Average round time:      %.3f seconds' % \
                     ((clock() - roundtime)/(i+1))
                 print
+            else:
+                print '%d done in %.3f seconds' % (i+1, (clock() - roundstarttime))
         self.roundtime = (clock() - roundtime) / self.rounds
         print
 
     def print_stat(self, compare_to=None, hidenoise=0):
 
         if not compare_to:
-            print '%-30s      per run    per oper.   overhead' % 'Tests:'
-            print '-'*72
+            print '%-30s       min run    avg run    per oprn  overhead' % 'Tests:'
+            print '-'*77
             tests = self.tests.items()
             tests.sort()
+            totalmintime = 0
             for name,t in tests:
-                avg,op_avg,ov_avg = t.stat()
-                print '%30s: %10.2f ms %7.2f us %7.2f ms' % \
-                      (name,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0)
-            print '-'*72
-            print '%30s: %10.2f ms' % \
-                  ('Average round time',self.roundtime * 1000.0)
+                mintime,avg,op_avg,ov_avg = t.stat()
+                totalmintime += mintime
+                print '%30s: %9.2f ms %9.2f ms %6.2f us  %6.2f' % \
+                      (name,mintime*1000.0,avg*1000.0,op_avg*1000000.0,ov_avg*1000.0)
+            print '-'*77
+            print '%30s: %9.2f ms' % \
+                  ('Notional minimum round time', totalmintime * 1000.0)
 
         else:
-            print '%-30s      per run    per oper.    diff *)' % \
+            print 'Comparing with: %s (rounds=%i, warp=%i)' % \
+                  (compare_to.name,compare_to.rounds,compare_to.warp)
+            print '%-30s      min run     cmp run     avg run      diff' % \
                   'Tests:'
-            print '-'*72
+            print '-'*77
             tests = self.tests.items()
             tests.sort()
             compatible = 1
+            totalmintime = other_totalmintime = 0
             for name, t in tests:
-                avg,op_avg,ov_avg = t.stat()
+                mintime, avg, op_avg, ov_avg = t.stat()
+                totalmintime += mintime
                 try:
                     other = compare_to.tests[name]
                 except KeyError:
                     other = None
                 if other and other.version == t.version and \
                    other.operations == t.operations:
-                    avg1,op_avg1,ov_avg1 = other.stat()
-                    qop_avg = (op_avg/op_avg1-1.0)*100.0
+                    mintime1, avg1, op_avg1, ov_avg1 = other.stat()
+                    other_totalmintime += mintime1
+                    diff = ((mintime*self.warp)/(mintime1*other.warp) - 1.0)*100.0
                     if hidenoise and abs(qop_avg) < 10:
-                        qop_avg = ''
+                        diff = ''
                     else:
-                        qop_avg = '%+7.2f%%' % qop_avg
+                        diff = '%+7.2f%%' % diff
                 else:
-                    qavg,qop_avg = 'n/a', 'n/a'
+                    qavg, diff = 'n/a', 'n/a'
                     compatible = 0
-                print '%30s: %10.2f ms %7.2f us  %8s' % \
-                      (name,avg*1000.0,op_avg*1000000.0,qop_avg)
-            print '-'*72
+                print '%30s: %8.2f ms %8.2f ms %8.2f ms  %8s' % \
+                      (name,mintime*1000.0,mintime1*1000.0 * compare_to.warp/self.warp, avg*1000.0,diff)
+            print '-'*77
+            #
+            # Summarise test results
+            #
             if compatible and compare_to.roundtime > 0 and \
                compare_to.version == self.version:
-                print '%30s: %10.2f ms             %+7.2f%%' % \
-                      ('Average round time',self.roundtime * 1000.0,
-                       ((self.roundtime*self.warp)/
-                        (compare_to.roundtime*compare_to.warp)-1.0)*100.0)
+                print '%30s: %8.2f ms %8.2f ms              %+7.2f%%' % \
+                      ('Notional minimum round time', totalmintime * 1000.0,
+                      other_totalmintime * 1000.0 * compare_to.warp/self.warp,
+                       ((totalmintime*self.warp)/
+                        (other_totalmintime*compare_to.warp)-1.0)*100.0)
             else:
-                print '%30s: %10.2f ms                  n/a' % \
-                      ('Average round time',self.roundtime * 1000.0)
-            print
-            print '*) measured against: %s (rounds=%i, warp=%i)' % \
-                  (compare_to.name,compare_to.rounds,compare_to.warp)
+                print '%30s: %9.2f ms                    n/a' % \
+                      ('Notional minimum round time', totalmintime * 1000.0)
         print
 
 def print_machine():
@@ -339,7 +374,12 @@ class PyBenchCmdline(Application):
                SwitchOption('-S','show statistics of benchmarks',0),
                ArgumentOption('-w','set warp factor to arg',Setup.Warp_factor),
                SwitchOption('-d','hide noise in compares', 0),
+               SwitchOption('-v','verbose output (not recommended)', 0),
                SwitchOption('--no-gc','disable garbage collection', 0),
+               SwitchOption('--no-syscheck',
+                    '"disable" sys check interval (set to sys.maxint)', 0),
+               ArgumentOption('-t', 'tests containing substring', ''),
+               ArgumentOption('-C', 'number of calibration runs (default 0)', '')
                ]
 
     about = """\
@@ -380,6 +420,11 @@ python pybench.py -s p15 -c p14
         hidenoise = self.values['-d']
         warp = self.values['-w']
         nogc = self.values['--no-gc']
+        limitnames = self.values['-t']
+        verbose = self.verbose
+        nosyscheck = self.values['--no-syscheck']
+        
+        print 'PYBENCH',__version__
 
         # Switch off GC
         if nogc:
@@ -390,8 +435,13 @@ python pybench.py -s p15 -c p14
             else:
                 if self.values['--no-gc']:
                     gc.disable()
+                    print 'NO GC'
+
+        # maximise sys check interval
+        if nosyscheck:
+            sys.setcheckinterval(sys.maxint)
+            print 'CHECKINTERVAL =', sys.maxint
 
-        print 'PYBENCH',__version__
         print
 
         if not compare_to:
@@ -436,9 +486,9 @@ python pybench.py -s p15 -c p14
         # Create benchmark object
         bench = Benchmark()
         bench.rounds = rounds
-        bench.load_tests(Setup,warp)
+        bench.load_tests(Setup, warp, limitnames, verbose)
         try:
-            bench.run()
+            bench.run(verbose)
         except KeyboardInterrupt:
             print
             print '*** KeyboardInterrupt -- Aborting'