Store a destination step in the block descriptor

At the moment, this just saves a memory reference in the GC inner loop
(worth a percent or two of GC time).  Later, it will hopefully let me
experiment with partial steps, and simplifying the generation/step
infrastructure.

diff --git a/includes/rts/storage/Block.h b/includes/rts/storage/Block.h
index 849f99f..e99a03e 100644 (file)
--- a/includes/rts/storage/Block.h
+++ b/includes/rts/storage/Block.h
@@ -48,22 +48,26 @@
 
 #ifndef CMINUSMINUS
 typedef struct bdescr_ {
-  StgPtr start;                        /* start addr of memory */
-  StgPtr free;                 /* first free byte of memory */
-  struct bdescr_ *link;                /* used for chaining blocks together */
-  union { 
-      struct bdescr_ *back;    /* used (occasionally) for doubly-linked lists*/
-      StgWord *bitmap;
-      StgPtr  scan;             /* scan pointer for copying GC */
-  } u;
-  unsigned int gen_no;         /* generation */
-  struct step_ *step;          /* step */
-  StgWord32 blocks;            /* no. of blocks (if grp head, 0 otherwise) */
-  StgWord32 flags;              /* block is in to-space */
+    StgPtr start;              /* start addr of memory */
+    StgPtr free;               /* first free byte of memory */
+    struct bdescr_ *link;      /* used for chaining blocks together */
+    union {
+        struct bdescr_ *back;  /* used (occasionally) for doubly-linked lists*/
+        StgWord *bitmap;
+        StgPtr  scan;           /* scan pointer for copying GC */
+    } u;
+
+    struct step_ *step;                /* step */
+    struct step_ *dest;                /* destination step */
+
+    StgWord32 blocks;          /* no. of blocks (if grp head, 0 otherwise) */
+
+    StgWord16 gen_no;
+    StgWord16 flags;            /* block flags, see below */
 #if SIZEOF_VOID_P == 8
-  StgWord32 _padding[2];
+    StgWord32 _padding[2];
 #else
-  StgWord32 _padding[0];
+    StgWord32 _padding[0];
 #endif
 } bdescr;
 #endif

diff --git a/includes/rts/storage/GC.h b/includes/rts/storage/GC.h
index b30582d..aa05313 100644 (file)
--- a/includes/rts/storage/GC.h
+++ b/includes/rts/storage/GC.h
@@ -214,4 +214,11 @@ void dirty_MUT_VAR(StgRegTable *reg, StgClosure *p);
 /* (needed when dynamic libraries are used). */
 extern rtsBool keepCAFs;
 
+INLINE_HEADER void initBdescr(bdescr *bd, step *step)
+{
+    bd->step   = step;
+    bd->gen_no = step->gen_no;
+    bd->dest   = step->to;
+}
+
 #endif /* RTS_STORAGE_GC_H */

diff --git a/rts/Arena.c b/rts/Arena.c
index 7fc49f4..e636de4 100644 (file)
--- a/rts/Arena.c
+++ b/rts/Arena.c
@@ -86,6 +86,7 @@ arenaAlloc( Arena *arena, size_t size )
 
        bd->gen_no  = 0;
        bd->step    = NULL;
+       bd->dest    = NULL;
        bd->flags   = 0;
        bd->free    = bd->start;
        bd->link    = arena->current;

diff --git a/rts/Schedule.c b/rts/Schedule.c
index dbee436..c377974 100644 (file)
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -1118,8 +1118,8 @@ scheduleHandleHeapOverflow( Capability *cap, StgTSO *t )
            { 
                bdescr *x;
                for (x = bd; x < bd + blocks; x++) {
-                   x->step = cap->r.rNursery;
-                   x->gen_no = 0;
+                    initBdescr(x,cap->r.rNursery);
+                    x->free = x->start;
                    x->flags = 0;
                }
            }

diff --git a/rts/Updates.h b/rts/Updates.h
index 988fb60..bf4f89d 100644 (file)
--- a/rts/Updates.h
+++ b/rts/Updates.h
@@ -191,7 +191,7 @@ no_slop:
     StgInd_indirectee(p1) = p2;                                        \
     prim %write_barrier() [];                                  \
     bd = Bdescr(p1);                                           \
-    if (bdescr_gen_no(bd) != 0 :: CInt) {                      \
+    if (bdescr_gen_no(bd) != 0 :: bits16) {                    \
       recordMutableCap(p1, TO_W_(bdescr_gen_no(bd)), R1);      \
       SET_INFO(p1, stg_IND_OLDGEN_info);                       \
       LDV_RECORD_CREATE(p1);                                   \

diff --git a/rts/sm/BlockAlloc.c b/rts/sm/BlockAlloc.c
index bf7a55e..d30d29b 100644 (file)
--- a/rts/sm/BlockAlloc.c
+++ b/rts/sm/BlockAlloc.c
@@ -59,6 +59,7 @@ static void  initMBlock(void *mblock);
      bd->flags
      bd->gen_no
      bd->step
+     bd->dest
 
   Exceptions: we don't maintain invariants for all the blocks within a
   group on the free list, because it is expensive to modify every

diff --git a/rts/sm/Evac.c b/rts/sm/Evac.c
index 3212ce5..9836e51 100644 (file)
--- a/rts/sm/Evac.c
+++ b/rts/sm/Evac.c
@@ -282,8 +282,7 @@ evacuate_large(StgPtr p)
   ws = &gct->steps[new_stp->abs_no];
 
   bd->flags |= BF_EVACUATED;
-  bd->step = new_stp;
-  bd->gen_no = new_stp->gen_no;
+  initBdescr(bd, new_stp);
 
   // If this is a block of pinned objects, we don't have to scan
   // these objects, because they aren't allowed to contain any
@@ -505,7 +504,7 @@ loop:
       return;
   }
       
-  stp = bd->step->to;
+  stp = bd->dest;
 
   info = q->header.info;
   if (IS_FORWARDING_PTR(info))
@@ -1069,7 +1068,7 @@ bale_out:
     // check whether it was updated in the meantime.
     *q = (StgClosure *)p;
     if (evac) {
-        copy(q,(const StgInfoTable *)info_ptr,(StgClosure *)p,THUNK_SELECTOR_sizeW(),bd->step->to);
+        copy(q,(const StgInfoTable *)info_ptr,(StgClosure *)p,THUNK_SELECTOR_sizeW(),bd->dest);
     }
     unchain_thunk_selectors(prev_thunk_selector, *q);
     return;

diff --git a/rts/sm/GCUtils.c b/rts/sm/GCUtils.c
index 70c53cb..7e99e29 100644 (file)
--- a/rts/sm/GCUtils.c
+++ b/rts/sm/GCUtils.c
@@ -269,8 +269,7 @@ alloc_todo_block (step_workspace *ws, nat size)
         } else {
             bd = allocBlock_sync();
         }
-        bd->step = ws->step;
-        bd->gen_no = ws->step->gen_no;
+        initBdescr(bd, ws->step);
         bd->flags = BF_EVACUATED;
         bd->u.scan = bd->free = bd->start;
     }

diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c
index f0506cd..73ef53f 100644 (file)
--- a/rts/sm/Storage.c
+++ b/rts/sm/Storage.c
@@ -411,8 +411,7 @@ allocNursery (step *stp, bdescr *tail, nat blocks)
        if (tail != NULL) {
            tail->u.back = bd;
        }
-       bd->step = stp;
-       bd->gen_no = 0;
+        initBdescr(bd, stp);
        bd->flags = 0;
        bd->free = bd->start;
        tail = bd;
@@ -612,8 +611,7 @@ allocateInGen (generation *g, lnat n)
        dbl_link_onto(bd, &stp->large_objects);
        stp->n_large_blocks += bd->blocks; // might be larger than req_blocks
        alloc_blocks += bd->blocks;
-       bd->gen_no  = g->no;
-       bd->step = stp;
+        initBdescr(bd, stp);
        bd->flags = BF_LARGE;
        bd->free = bd->start + n;
        ret = bd->start;
@@ -624,8 +622,7 @@ allocateInGen (generation *g, lnat n)
         bd = stp->blocks;
        if (bd == NULL || bd->free + n > bd->start + BLOCK_SIZE_W) {
             bd = allocBlock();
-            bd->gen_no = g->no;
-            bd->step = stp;
+            initBdescr(bd, stp);
             bd->flags = 0;
             bd->link = stp->blocks;
             stp->blocks = bd;
@@ -676,8 +673,7 @@ splitLargeBlock (bdescr *bd, nat blocks)
 
     dbl_link_onto(new_bd, &g0s0->large_objects);
     g0s0->n_large_blocks += new_bd->blocks;
-    new_bd->gen_no  = g0s0->no;
-    new_bd->step    = g0s0;
+    initBdescr(new_bd, g0s0);
     new_bd->flags   = BF_LARGE;
     new_bd->free    = bd->free;
     ASSERT(new_bd->free <= new_bd->start + new_bd->blocks * BLOCK_SIZE_W);
@@ -733,8 +729,7 @@ allocateLocal (Capability *cap, lnat n)
             bd = allocBlock();
             cap->r.rNursery->n_blocks++;
             RELEASE_SM_LOCK;
-            bd->gen_no = 0;
-            bd->step = cap->r.rNursery;
+            initBdescr(bd, cap->r.rNursery);
             bd->flags = 0;
             // NO: alloc_blocks++;
             // calcAllocated() uses the size of the nursery, and we've
@@ -807,8 +802,7 @@ allocatePinned( lnat n )
        pinned_object_block = bd = allocBlock();
        dbl_link_onto(bd, &g0s0->large_objects);
        g0s0->n_large_blocks++;
-       bd->gen_no = 0;
-       bd->step   = g0s0;
+        initBdescr(bd, g0s0);
        bd->flags  = BF_PINNED | BF_LARGE;
        bd->free   = bd->start;
        alloc_blocks++;