From: Simon Marlow Date: Wed, 31 Oct 2007 14:42:30 +0000 (+0000) Subject: GC refactoring: change evac_gen to evac_step X-Git-Tag: Before_cabalised-GHC~300 X-Git-Url: http://git.megacz.com/?a=commitdiff_plain;h=c357244371ca5eeaa0d56a6bd349e26a3b16b7dc;p=ghc-hetmet.git GC refactoring: change evac_gen to evac_step By establishing an ordering on step pointers, we can simplify the test (stp->gen_no < evac_gen) to (stp < evac_step) which is common in evacuate(). --- diff --git a/rts/sm/Evac.c b/rts/sm/Evac.c index 518b383..14b80a7 100644 --- a/rts/sm/Evac.c +++ b/rts/sm/Evac.c @@ -39,9 +39,9 @@ alloc_for_copy (nat size, step *stp) * evacuate to an older generation, adjust it here (see comment * by evacuate()). */ - if (stp->gen_no < gct->evac_gen) { + if (stp < gct->evac_step) { if (gct->eager_promotion) { - stp = &generations[gct->evac_gen].steps[0]; + stp = gct->evac_step; } else { gct->failed_to_evac = rtsTrue; } @@ -75,9 +75,9 @@ alloc_for_copy_noscav (nat size, step *stp) * evacuate to an older generation, adjust it here (see comment * by evacuate()). */ - if (stp->gen_no < gct->evac_gen) { + if (stp < gct->evac_step) { if (gct->eager_promotion) { - stp = &generations[gct->evac_gen].steps[0]; + stp = gct->evac_step; } else { gct->failed_to_evac = rtsTrue; } @@ -113,7 +113,7 @@ copy_tag(StgClosure **p, StgClosure *src, nat size, step *stp,StgWord tag) } while (info == (W_)&stg_WHITEHOLE_info); if (info == (W_)&stg_EVACUATED_info) { src->header.info = (const StgInfoTable *)info; - return evacuate(src); // does the failed_to_evac stuff + return evacuate(p); // does the failed_to_evac stuff } #else info = (W_)src->header.info; @@ -169,13 +169,13 @@ copy_noscav_tag(StgClosure **p, StgClosure *src, nat size, step *stp, StgWord ta } while (info == (W_)&stg_WHITEHOLE_info); if (info == (W_)&stg_EVACUATED_info) { src->header.info = (const StgInfoTable *)info; - return evacuate(src); // does the failed_to_evac stuff + return evacuate(p); // does the failed_to_evac stuff } #else info = (W_)src->header.info; src->header.info = &stg_EVACUATED_info; #endif - + to = alloc_for_copy_noscav(size,stp); tagged_to = (StgPtr)TAG_CLOSURE(tag,(StgClosure*)to); *p = (StgClosure *)tagged_to; @@ -220,7 +220,7 @@ copyPart(StgClosure **p, StgClosure *src, nat size_to_reserve, nat size_to_copy, } while (info == (W_)&stg_WHITEHOLE_info); if (info == (W_)&stg_EVACUATED_info) { src->header.info = (const StgInfoTable *)info; - return evacuate(src); // does the failed_to_evac stuff + return evacuate(p); // does the failed_to_evac stuff } #else info = (W_)src->header.info; @@ -296,7 +296,7 @@ evacuate_large(StgPtr p) /* Don't forget to set the gct->failed_to_evac flag if we didn't get * the desired destination (see comments in evacuate()). */ - if (bd->gen_no < gct->evac_gen) { + if (bd->step < gct->evac_step) { gct->failed_to_evac = rtsTrue; TICK_GC_FAILED_PROMOTION(); } @@ -320,9 +320,9 @@ evacuate_large(StgPtr p) /* link it on to the evacuated large object list of the destination step */ stp = bd->step->to; - if (stp->gen_no < gct->evac_gen) { + if (stp < gct->evac_step) { if (gct->eager_promotion) { - stp = &generations[gct->evac_gen].steps[0]; + stp = gct->evac_step; } else { gct->failed_to_evac = rtsTrue; } @@ -342,22 +342,22 @@ evacuate_large(StgPtr p) This is called (eventually) for every live object in the system. The caller to evacuate specifies a desired generation in the - gct->evac_gen thread-lock variable. The following conditions apply to + gct->evac_step thread-local variable. The following conditions apply to evacuating an object which resides in generation M when we're collecting up to generation N - if M >= gct->evac_gen + if M >= gct->evac_step if M > N do nothing else evac to step->to - if M < gct->evac_gen evac to gct->evac_gen, step 0 + if M < gct->evac_step evac to gct->evac_step, step 0 if the object is already evacuated, then we check which generation it now resides in. - if M >= gct->evac_gen do nothing - if M < gct->evac_gen set gct->failed_to_evac flag to indicate that we - didn't manage to evacuate this object into gct->evac_gen. + if M >= gct->evac_step do nothing + if M < gct->evac_step set gct->failed_to_evac flag to indicate that we + didn't manage to evacuate this object into gct->evac_step. OPTIMISATION NOTES: @@ -472,10 +472,10 @@ loop: if (bd->gen_no > N) { /* Can't evacuate this object, because it's in a generation * older than the ones we're collecting. Let's hope that it's - * in gct->evac_gen or older, or we will have to arrange to track + * in gct->evac_step or older, or we will have to arrange to track * this pointer using the mutable list. */ - if (bd->gen_no < gct->evac_gen) { + if (bd->step < gct->evac_step) { // nope gct->failed_to_evac = rtsTrue; TICK_GC_FAILED_PROMOTION(); @@ -491,7 +491,7 @@ loop: * object twice, for example). */ if (bd->flags & BF_EVACUATED) { - if (bd->gen_no < gct->evac_gen) { + if (bd->step < gct->evac_step) { gct->failed_to_evac = rtsTrue; TICK_GC_FAILED_PROMOTION(); } @@ -664,7 +664,7 @@ loop: case EVACUATED: /* Already evacuated, just return the forwarding address. - * HOWEVER: if the requested destination generation (gct->evac_gen) is + * HOWEVER: if the requested destination generation (gct->evac_step) is * older than the actual generation (because the object was * already evacuated to a younger generation) then we have to * set the gct->failed_to_evac flag to indicate that we couldn't @@ -682,8 +682,8 @@ loop: { StgClosure *e = ((StgEvacuated*)q)->evacuee; *p = e; - if (gct->evac_gen > 0 && stp->gen_no < gct->evac_gen) { // optimisation - if (HEAP_ALLOCED(e) && Bdescr((P_)e)->gen_no < gct->evac_gen) { + if (stp < gct->evac_step) { // optimisation + if (HEAP_ALLOCED(e) && Bdescr((P_)e)->step < gct->evac_step) { gct->failed_to_evac = rtsTrue; TICK_GC_FAILED_PROMOTION(); } diff --git a/rts/sm/GC.c b/rts/sm/GC.c index d064f1d..4ca0c9d 100644 --- a/rts/sm/GC.c +++ b/rts/sm/GC.c @@ -335,11 +335,11 @@ GarbageCollect ( rtsBool force_major_gc ) } // follow roots from the CAF list (used by GHCi) - gct->evac_gen = 0; + gct->evac_step = 0; markCAFs(mark_root); // follow all the roots that the application knows about. - gct->evac_gen = 0; + gct->evac_step = 0; GetRoots(mark_root); // Mark the weak pointer list, and prepare to detect dead weak pointers. @@ -1287,7 +1287,7 @@ init_uncollected_gen (nat g, nat threads) static void init_gc_thread (gc_thread *t) { - t->evac_gen = 0; + t->evac_step = 0; t->failed_to_evac = rtsFalse; t->eager_promotion = rtsTrue; t->thunk_selector_depth = 0; diff --git a/rts/sm/GC.h b/rts/sm/GC.h index 013a901..d45efb9 100644 --- a/rts/sm/GC.h +++ b/rts/sm/GC.h @@ -128,7 +128,7 @@ typedef struct gc_thread_ { // -------------------- // evacuate flags - nat evac_gen; // Youngest generation that objects + step *evac_step; // Youngest generation that objects // should be evacuated to in // evacuate(). (Logically an // argument to evacuate, but it's diff --git a/rts/sm/MarkWeak.c b/rts/sm/MarkWeak.c index 62ac8e0..eca5c54 100644 --- a/rts/sm/MarkWeak.c +++ b/rts/sm/MarkWeak.c @@ -109,7 +109,7 @@ traverseWeakPtrList(void) /* doesn't matter where we evacuate values/finalizers to, since * these pointers are treated as roots (iff the keys are alive). */ - gct->evac_gen = 0; + gct->evac_step = 0; last_w = &old_weak_ptr_list; for (w = old_weak_ptr_list; w != NULL; w = next_w) { diff --git a/rts/sm/Scav.c b/rts/sm/Scav.c index 080c750..71c2be7 100644 --- a/rts/sm/Scav.c +++ b/rts/sm/Scav.c @@ -254,11 +254,11 @@ scavenge_AP (StgAP *ap) /* ----------------------------------------------------------------------------- Scavenge a block from the given scan pointer up to bd->free. - evac_gen is set by the caller to be either zero (for a step in a + evac_step is set by the caller to be either zero (for a step in a generation < N) or G where G is the generation of the step being scavenged. - We sometimes temporarily change evac_gen back to zero if we're + We sometimes temporarily change evac_step back to zero if we're scavenging a mutable object where eager promotion isn't such a good idea. -------------------------------------------------------------------------- */ @@ -268,15 +268,15 @@ scavenge_block (bdescr *bd, StgPtr scan) { StgPtr p, q; StgInfoTable *info; - nat saved_evac_gen; + step *saved_evac_step; p = scan; debugTrace(DEBUG_gc, "scavenging block %p (gen %d, step %d) @ %p", bd->start, bd->gen_no, bd->step->no, scan); - gct->evac_gen = bd->gen_no; - saved_evac_gen = gct->evac_gen; + gct->evac_step = bd->step; + saved_evac_step = gct->evac_step; gct->failed_to_evac = rtsFalse; // we might be evacuating into the very object that we're @@ -573,11 +573,11 @@ scavenge_block (bdescr *bd, StgPtr scan) case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTVarWatchQueue); break; @@ -586,10 +586,10 @@ scavenge_block (bdescr *bd, StgPtr scan) case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTVar); break; @@ -598,11 +598,11 @@ scavenge_block (bdescr *bd, StgPtr scan) case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTRecHeader); break; @@ -613,14 +613,14 @@ scavenge_block (bdescr *bd, StgPtr scan) StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTRecChunk); break; @@ -629,10 +629,10 @@ scavenge_block (bdescr *bd, StgPtr scan) case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgAtomicInvariant); break; @@ -641,11 +641,11 @@ scavenge_block (bdescr *bd, StgPtr scan) case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgInvariantCheckQueue); break; @@ -687,10 +687,10 @@ scavenge_mark_stack(void) { StgPtr p, q; StgInfoTable *info; - nat saved_evac_gen; + step *saved_evac_step; - gct->evac_gen = oldest_gen->no; - saved_evac_gen = gct->evac_gen; + gct->evac_step = &oldest_gen->steps[0]; + saved_evac_step = gct->evac_step; linear_scan: while (!mark_stack_empty()) { @@ -939,11 +939,11 @@ linear_scan: case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -951,10 +951,10 @@ linear_scan: case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -964,14 +964,14 @@ linear_scan: StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -979,11 +979,11 @@ linear_scan: case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -991,10 +991,10 @@ linear_scan: case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1002,11 +1002,11 @@ linear_scan: case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1018,8 +1018,8 @@ linear_scan: if (gct->failed_to_evac) { gct->failed_to_evac = rtsFalse; - if (gct->evac_gen > 0) { - recordMutableGen_GC((StgClosure *)q, &generations[gct->evac_gen]); + if (gct->evac_step) { + recordMutableGen_GC((StgClosure *)q, gct->evac_step->gen); } } @@ -1079,7 +1079,7 @@ static rtsBool scavenge_one(StgPtr p) { const StgInfoTable *info; - nat saved_evac_gen = gct->evac_gen; + step *saved_evac_step = gct->evac_step; rtsBool no_luck; ASSERT(LOOKS_LIKE_CLOSURE_PTR(p)); @@ -1271,11 +1271,11 @@ scavenge_one(StgPtr p) case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1283,10 +1283,10 @@ scavenge_one(StgPtr p) case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1294,11 +1294,11 @@ scavenge_one(StgPtr p) case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1308,14 +1308,14 @@ scavenge_one(StgPtr p) StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1323,10 +1323,10 @@ scavenge_one(StgPtr p) case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1334,11 +1334,11 @@ scavenge_one(StgPtr p) case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_gen = 0; + gct->evac_step = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_gen = saved_evac_gen; + gct->evac_step = saved_evac_step; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1413,7 +1413,7 @@ scavenge_mutable_list(generation *gen) bd = gen->saved_mut_list; - gct->evac_gen = gen->no; + gct->evac_step = &gen->steps[0]; for (; bd != NULL; bd = bd->link) { for (q = bd->start; q < bd->free; q++) { p = (StgPtr)*q; @@ -1497,7 +1497,7 @@ scavenge_static(void) /* Always evacuate straight to the oldest generation for static * objects */ - gct->evac_gen = oldest_gen->no; + gct->evac_step = &oldest_gen->steps[0]; /* keep going until we've scavenged all the objects on the linked list... */ @@ -1774,9 +1774,9 @@ scavenge_stack(StgPtr p, StgPtr stack_end) /*----------------------------------------------------------------------------- scavenge the large object list. - evac_gen set by caller; similar games played with evac_gen as with + evac_step set by caller; similar games played with evac_step as with scavenge() - see comment at the top of scavenge(). Most large - objects are (repeatedly) mutable, so most of the time evac_gen will + objects are (repeatedly) mutable, so most of the time evac_step will be zero. --------------------------------------------------------------------------- */ @@ -1786,7 +1786,7 @@ scavenge_large (step_workspace *ws) bdescr *bd; StgPtr p; - gct->evac_gen = ws->stp->gen_no; + gct->evac_step = ws->stp; bd = ws->todo_large_objects; diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c index 9b86b43..c441826 100644 --- a/rts/sm/Storage.c +++ b/rts/sm/Storage.c @@ -103,6 +103,7 @@ initStorage( void ) { nat g, s; generation *gen; + step *step_arr; if (generations != NULL) { // multi-init protection @@ -146,6 +147,15 @@ initStorage( void ) * sizeof(struct generation_), "initStorage: gens"); + /* allocate all the steps into an array. It is important that we do + it this way, because we need the invariant that two step pointers + can be directly compared to see which is the oldest. + Remember that the last generation has only one step. */ + step_arr = stgMallocBytes(sizeof(struct step_) + * (1 + ((RtsFlags.GcFlags.generations - 1) + * RtsFlags.GcFlags.steps)), + "initStorage: steps"); + /* Initialise all generations */ for(g = 0; g < RtsFlags.GcFlags.generations; g++) { gen = &generations[g]; @@ -166,21 +176,19 @@ initStorage( void ) /* Oldest generation: one step */ oldest_gen->n_steps = 1; - oldest_gen->steps = - stgMallocBytes(1 * sizeof(struct step_), "initStorage: last step"); + oldest_gen->steps = step_arr + (RtsFlags.GcFlags.generations - 1) + * RtsFlags.GcFlags.steps; /* set up all except the oldest generation with 2 steps */ for(g = 0; g < RtsFlags.GcFlags.generations-1; g++) { generations[g].n_steps = RtsFlags.GcFlags.steps; - generations[g].steps = - stgMallocBytes (RtsFlags.GcFlags.steps * sizeof(struct step_), - "initStorage: steps"); + generations[g].steps = step_arr + g * RtsFlags.GcFlags.steps; } } else { /* single generation, i.e. a two-space collector */ g0->n_steps = 1; - g0->steps = stgMallocBytes (sizeof(struct step_), "initStorage: steps"); + g0->steps = step_arr; } #ifdef THREADED_RTS @@ -264,10 +272,7 @@ exitStorage (void) void freeStorage (void) { - nat g; - - for(g = 0; g < RtsFlags.GcFlags.generations; g++) - stgFree(generations[g].steps); + stgFree(g0s0); // frees all the steps stgFree(generations); freeAllMBlocks(); #if defined(THREADED_RTS)