GC: rearrange storage to reduce memory accesses in the inner loop

[ghc-hetmet.git] / includes / Storage.h

diff --git a/includes/Storage.h b/includes/Storage.h
index 48b7ec3..28225d7 100644 (file)
--- a/includes/Storage.h
+++ b/includes/Storage.h
@@ -53,7 +53,8 @@
  * ------------------------------------------------------------------------- */
 
 typedef struct step_ {
-    unsigned int         no;           // step number
+    unsigned int         no;           // step number in this generation
+    unsigned int         abs_no;       // absolute step number
     int                  is_compacted; // compact this step? (old gen only)
 
     struct generation_ * gen;          // generation this step belongs to
@@ -67,28 +68,34 @@ typedef struct step_ {
     bdescr *             large_objects;         // large objects (doubly linked)
     unsigned int         n_large_blocks; // no. of blocks used by large objs
 
+
     // ------------------------------------
     // Fields below are used during GC only
 
     // During GC, if we are collecting this step, blocks and n_blocks
     // are copied into the following two fields.  After GC, these blocks
     // are freed.
-    bdescr *     old_blocks;           // bdescr of first from-space block
-    unsigned int n_old_blocks;         // number of blocks in from-space
-    
-    bdescr *     todos;                        // blocks waiting to be scavenged
-    unsigned int n_todos;               // count of above
 
 #if defined(THREADED_RTS)
+    char pad[128];                      // make sure the following is
+                                        // on a separate cache line.
     SpinLock     sync_todo;             // lock for todos
     SpinLock     sync_large_objects;    // lock for large_objects
                                         //    and scavenged_large_objects
 #endif
 
+    bdescr *     old_blocks;           // bdescr of first from-space block
+    unsigned int n_old_blocks;         // number of blocks in from-space
+    
+    bdescr *     todos;                        // blocks waiting to be scavenged
+    unsigned int n_todos;               // count of above
+
     bdescr *     scavenged_large_objects;  // live large objs after GC (d-link)
     unsigned int n_scavenged_large_blocks; // size (not count) of above
 
     bdescr *     bitmap;               // bitmap for compacting collection
+
+
 } step;
 
 
@@ -112,6 +119,8 @@ extern generation * RTS_VAR(generations);
 extern generation * RTS_VAR(g0);
 extern step * RTS_VAR(g0s0);
 extern generation * RTS_VAR(oldest_gen);
+extern step * RTS_VAR(all_steps);
+extern nat total_steps;
 
 /* -----------------------------------------------------------------------------
    Initialisation / De-initialisation
@@ -173,9 +182,6 @@ extern bdescr * RTS_VAR(small_alloc_list);
 extern bdescr * RTS_VAR(large_alloc_list);
 extern bdescr * RTS_VAR(pinned_object_block);
 
-extern StgPtr RTS_VAR(alloc_Hp);
-extern StgPtr RTS_VAR(alloc_HpLim);
-
 extern nat RTS_VAR(alloc_blocks);
 extern nat RTS_VAR(alloc_blocks_lim);
 
@@ -189,6 +195,9 @@ doYouWantToGC( void )
 extern void *allocateExec (nat bytes);
 extern void freeExec (void *p);
 
+/* for splitting blocks groups in two */
+extern bdescr * splitLargeBlock (bdescr *bd, nat blocks);
+
 /* -----------------------------------------------------------------------------
    Performing Garbage Collection
 
@@ -262,11 +271,28 @@ recordMutableGenLock(StgClosure *p, generation *gen)
     RELEASE_SM_LOCK;
 }
 
+extern bdescr *allocBlock_sync(void);
+
+// Version of recordMutableGen() for use in parallel GC.  The same as
+// recordMutableGen(), except that we surround it with a spinlock and
+// call the spinlock version of allocBlock().
 INLINE_HEADER void
 recordMutableGen_GC(StgClosure *p, generation *gen)
 {
+    bdescr *bd;
+
     ACQUIRE_SPIN_LOCK(&recordMutableGen_sync);
-    recordMutableGen(p,gen);
+
+    bd = gen->mut_list;
+    if (bd->free >= bd->start + BLOCK_SIZE_W) {
+       bdescr *new_bd;
+       new_bd = allocBlock_sync();
+       new_bd->link = bd;
+       bd = new_bd;
+       gen->mut_list = bd;
+    }
+    *bd->free++ = (StgWord)p;
+
     RELEASE_SPIN_LOCK(&recordMutableGen_sync);
 }
 
@@ -522,11 +548,13 @@ extern void         GetRoots     ( evac_fn evac );
 extern ullong RTS_VAR(total_allocated);
 
 extern lnat calcAllocated  ( void );
-extern lnat calcLive       ( void );
+extern lnat calcLiveBlocks ( void );
+extern lnat calcLiveWords  ( void );
+extern lnat countOccupied  ( bdescr *bd );
 extern lnat calcNeeded     ( void );
 
 #if defined(DEBUG)
-extern void memInventory(void);
+extern void memInventory(rtsBool show);
 extern void checkSanity(void);
 extern nat  countBlocks(bdescr *);
 extern void checkNurserySanity( step *stp );