[commit: ghc] master: Parallelise clearNurseries() in the parallel GC (713cf47)

[Simon Marlow]

Repository : ssh://darcs.haskell.org//srv/darcs/ghc

On branch  : master

http://hackage.haskell.org/trac/ghc/changeset/713cf473de8a2ad7d0b8195d78860c25fec41839

>---------------------------------------------------------------

commit 713cf473de8a2ad7d0b8195d78860c25fec41839
Author: Simon Marlow <marlowsd at gmail.com>
Date:   Tue Jul 10 09:57:53 2012 +0100

    Parallelise clearNurseries() in the parallel GC
    
    The clearNurseries() operation resets the free pointer in each nursery
    block to the start of the block, emptying the nursery.  In the
    parallel GC this was done on the main GC thread, but that's bad
    because it accesses the bdescr of every nursery block, and move all
    those cache lines onto the CPU of the main GC thread.  With large
    nurseries, this can be especially bad.  So instead we want to clear
    each nursery in its local GC thread.
    
    Thanks to Andreas Voellmy <andreas.voellmy at gmail.com> for idenitfying
    the issue.
    
    After this change and the previous patch to make the last GC a major
    one, I see these results for nofib/parallel on 8 cores:
    
       blackscholes          +0.0%     +0.0%     -3.7%     -3.3%     +0.3%
              coins          +0.0%     +0.0%     -5.1%     -5.0%     +0.4%
               gray          +0.0%     +0.0%     -4.5%     -2.1%     +0.8%
             mandel          +0.0%     -0.0%     -7.6%     -5.1%     -2.3%
            matmult          +0.0%     +5.5%     -2.8%     -1.9%     -5.8%
            minimax          +0.0%     +0.0%    -10.6%    -10.5%     +0.0%
              nbody          +0.0%     -4.4%     +0.0%      0.07     +0.0%
             parfib          +0.0%     +1.0%     +0.5%     +0.9%     +0.0%
            partree          +0.0%     +0.0%     -2.4%     -2.5%     +1.7%
               prsa          +0.0%     -0.2%     +1.8%     +4.2%     +0.0%
             queens          +0.0%     -0.0%     -1.8%     -1.4%     -4.8%
                ray          +0.0%     -0.6%    -18.5%    -17.8%     +0.0%
           sumeuler          +0.0%     -0.0%     -3.7%     -3.7%     +0.0%
          transclos          +0.0%     -0.0%    -25.7%    -26.6%     +0.0%
    --------------------------------------------------------------------------------
                Min          +0.0%     -4.4%    -25.7%    -26.6%     -5.8%
                Max          +0.0%     +5.5%     +1.8%     +4.2%     +1.7%
     Geometric Mean          +0.0%     +0.1%     -6.3%     -6.1%     -0.7%

>---------------------------------------------------------------

 rts/sm/GC.c       |   14 +++++++++++++-
 rts/sm/GCThread.h |    1 +
 rts/sm/Storage.c  |   23 ++++++++++-------------
 rts/sm/Storage.h  |    2 +-
 4 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index a6b8c4a..ab0ba64 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -629,7 +629,16 @@ GarbageCollect (rtsBool force_major_gc,
   }
 
   // Reset the nursery: make the blocks empty
-  allocated += clearNurseries();
+  if (n_gc_threads == 1) {
+      for (n = 0; n < n_capabilities; n++) {
+          allocated += clearNursery(&capabilities[n]);
+      }
+  } else {
+      gct->allocated = clearNursery(cap);
+      for (n = 0; n < n_capabilities; n++) {
+          allocated += gc_threads[n]->allocated;
+      }
+  }
 
   resize_nursery();
 
@@ -1094,6 +1103,8 @@ gcWorkerThread (Capability *cap)
 
     scavenge_until_all_done();
     
+    gct->allocated = clearNursery(cap);
+
 #ifdef THREADED_RTS
     // Now that the whole heap is marked, we discard any sparks that
     // were found to be unreachable.  The main GC thread is currently
@@ -1477,6 +1488,7 @@ init_gc_thread (gc_thread *t)
     t->failed_to_evac = rtsFalse;
     t->eager_promotion = rtsTrue;
     t->thunk_selector_depth = 0;
+    t->allocated = 0;
     t->copied = 0;
     t->scanned = 0;
     t->any_work = 0;
diff --git a/rts/sm/GCThread.h b/rts/sm/GCThread.h
index 60f7212..1b811e4 100644
--- a/rts/sm/GCThread.h
+++ b/rts/sm/GCThread.h
@@ -176,6 +176,7 @@ typedef struct gc_thread_ {
     // -------------------
     // stats
 
+    lnat allocated;          // result of clearNursery()
     lnat copied;
     lnat scanned;
     lnat any_work;
diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index 17798a2..18d317d 100644
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -496,22 +496,19 @@ allocNurseries (nat from, nat to)
     assignNurseriesToCapabilities(from, to);
 }
       
-lnat // words allocated
-clearNurseries (void)
+lnat
+clearNursery (Capability *cap)
 {
-    lnat allocated = 0;
-    nat i;
     bdescr *bd;
+    lnat allocated = 0;
 
-    for (i = 0; i < n_capabilities; i++) {
-        for (bd = nurseries[i].blocks; bd; bd = bd->link) {
-            allocated                       += (lnat)(bd->free - bd->start);
-            capabilities[i].total_allocated += (lnat)(bd->free - bd->start);
-            bd->free = bd->start;
-            ASSERT(bd->gen_no == 0);
-            ASSERT(bd->gen == g0);
-            IF_DEBUG(sanity,memset(bd->start, 0xaa, BLOCK_SIZE));
-        }
+    for (bd = nurseries[cap->no].blocks; bd; bd = bd->link) {
+        allocated            += (lnat)(bd->free - bd->start);
+        cap->total_allocated += (lnat)(bd->free - bd->start);
+        bd->free = bd->start;
+        ASSERT(bd->gen_no == 0);
+        ASSERT(bd->gen == g0);
+        IF_DEBUG(sanity,memset(bd->start, 0xaa, BLOCK_SIZE));
     }
 
     return allocated;
diff --git a/rts/sm/Storage.h b/rts/sm/Storage.h
index 44f39ee..9dffc18 100644
--- a/rts/sm/Storage.h
+++ b/rts/sm/Storage.h
@@ -81,7 +81,7 @@ void dirty_MVAR(StgRegTable *reg, StgClosure *p);
 extern nursery *nurseries;
 
 void     resetNurseries       ( void );
-lnat     clearNurseries       ( void );
+lnat     clearNursery         ( Capability *cap );
 void     resizeNurseries      ( nat blocks );
 void     resizeNurseriesFixed ( nat blocks );
 lnat     countNurseryBlocks   ( void );