[commit: testsuite] master: Allow perf bounds to be specified as base + percentage deviation (d183af3)

[Simon Marlow]

Repository : ssh://darcs.haskell.org//srv/darcs/testsuite

On branch  : master

http://hackage.haskell.org/trac/ghc/changeset/d183af39b5e1f74f28ebfe833151d42c50814f26

>---------------------------------------------------------------

commit d183af39b5e1f74f28ebfe833151d42c50814f26
Author: Simon Marlow <marlowsd at gmail.com>
Date:   Wed Feb 15 14:33:36 2012 +0000

    Allow perf bounds to be specified as base + percentage deviation
    
    This makes it much easier to update the bounds.  Instead of coming up
    with a suitable (min,max) pair, you just give e.g. (base, 10) to allow
    10% deviation from the base figure, which can be pasted from the
    error.
    
    e.g. previously:
    
    -      # expected value: 458700632 (amd64/Linux):
    -      if_wordsize(64,
    -          compiler_stats_num_field('bytes allocated', 440000000,
    -                                                      480000000)),
    
    now:
    
    +      if_wordsize(64, # sample from amd64/Linux 15/2/2012
    +          compiler_stats_range_field('bytes allocated', 360243576, 10)),
    
    Note: use stats_range_field rather than stats_num_field.  I left
    support for the old way for now so that we can do a gradual migration.
    
    (next I suppose we should make it so that 'make accept' works for perf
    tests, but that's for another day)

>---------------------------------------------------------------

 driver/testglobals.py |   12 ++++++++++++
 driver/testlib.py     |   47 ++++++++++++++++++++++++++++++++++++++++++++---
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/driver/testglobals.py b/driver/testglobals.py
index 6e079a7..9a4bfba 100644
--- a/driver/testglobals.py
+++ b/driver/testglobals.py
@@ -201,6 +201,18 @@ class TestOptions:
        # Elements of these lists should be things like
        # ('bytes allocated',
        #   9300000000,
+       #   10)
+       # To allow a 10% deviation from 9300000000.
+       self.compiler_stats_range_fields = {}
+       self.stats_range_fields = {}
+
+       # TODO: deprecate this in favour of compiler_stats_range_fields
+       #
+       # which -t numeric fields do we want to look at, and what bounds must
+       # they fall within?
+       # Elements of these lists should be things like
+       # ('bytes allocated',
+       #   9300000000,
        #   9400000000)
        self.compiler_stats_num_fields = {}
        self.stats_num_fields = {}
diff --git a/driver/testlib.py b/driver/testlib.py
index f877edb..cb4a765 100644
--- a/driver/testlib.py
+++ b/driver/testlib.py
@@ -253,6 +253,24 @@ def _compiler_stats_num_field( opts, f, x, y ):
 
 # -----
 
+def stats_range_field( field, min, max ):
+    return lambda opts, f=field, x=min, y=max: _stats_range_field(opts, f, x, y);
+
+def _stats_range_field( opts, f, x, y ):
+    # copy the dictionary, as the config gets shared between all tests
+    opts.stats_range_fields = opts.stats_range_fields.copy()
+    opts.stats_range_fields[f] = (x, y)
+
+def compiler_stats_range_field( field, min, max ):
+    return lambda opts, f=field, x=min, y=max: _compiler_stats_range_field(opts, f, x, y);
+
+def _compiler_stats_range_field( opts, f, x, y ):
+    # copy the dictionary, as the config gets shared between all tests
+    opts.compiler_stats_range_fields = opts.compiler_stats_range_fields.copy()
+    opts.compiler_stats_range_fields[f] = (x, y)
+
+# -----
+
 def skip_if_no_ghci(opts):
     if not ('ghci' in config.run_ways):
         opts.skip = 1
@@ -949,13 +967,34 @@ def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
 # -----------------------------------------------------------------------------
 # Check -t stats info
 
-def checkStats(stats_file, num_fields):
+def checkStats(stats_file, range_fields, num_fields):
     result = passed()
     if len(num_fields) > 0:
         f = open(in_testdir(stats_file))
         contents = f.read()
         f.close()
 
+        for (field, (expected, dev)) in range_fields.items():
+            m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
+            if m == None:
+                print 'Failed to find field: ', field
+                result = failBecause('no such stats field')
+            val = int(m.group(1))
+
+            min = expected * ((100 - float(dev))/100);
+            max = expected * ((100 + float(dev))/100);
+
+            if val < min:
+                print field, val, 'is more than ' + repr(dev) + '%'
+                print 'less than the exepected value', expected
+                print 'If this is because you have improved GHC, please'
+                print 'update the test so that GHC doesn\'t regress again'
+                result = failBecause('stat too good')
+            if val > max:
+                print field, val, 'is more than ' + repr(dev) + '% greater than the expected value,', expected, max
+                result = failBecause('stat not good enough')
+
+        # ToDo: remove all uses of this, and delete it
         for (field, (min, max)) in num_fields.items():
             m = re.search('\("' + field + '", "([0-9]+)"\)', contents)
             if m == None:
@@ -1058,7 +1097,8 @@ def simple_build( name, way, extra_hc_opts, should_fail, top_mod, link, addsuf,
 
     # ToDo: if the sub-shell was killed by ^C, then exit
 
-    statsResult = checkStats(stats_file, opts.compiler_stats_num_fields)
+    statsResult = checkStats(stats_file, opts.compiler_stats_range_fields
+                                       , opts.compiler_stats_num_fields)
 
     if badResult(statsResult):
         return statsResult
@@ -1149,7 +1189,8 @@ def simple_run( name, way, prog, args ):
         if check_prof and not check_prof_ok(name):
             return failBecause('bad profile')
 
-    return checkStats(stats_file, opts.stats_num_fields)
+    return checkStats(stats_file, opts.stats_range_fields
+                                , opts.stats_num_fields)
 
 def rts_flags(way):
     if (way == ''):