X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=rts%2Fsm%2FScav.c;h=4fa0a220e0dc8a8ff74d436f4baba6f2d3585691;hb=4207605cc8e035e05f651334ef1be90b098d305d;hp=4c75ed2799ea51bfb071869a482f28aa97813a6d;hpb=91a5df02a15c0a06b1d4a36a569e41b0624a1185;p=ghc-hetmet.git diff --git a/rts/sm/Scav.c b/rts/sm/Scav.c index 4c75ed2..4fa0a22 100644 --- a/rts/sm/Scav.c +++ b/rts/sm/Scav.c @@ -19,6 +19,7 @@ #include "GCThread.h" #include "GCUtils.h" #include "Compact.h" +#include "MarkStack.h" #include "Evac.h" #include "Scav.h" #include "Apply.h" @@ -37,6 +38,7 @@ static void scavenge_large_bitmap (StgPtr p, # define evacuate(a) evacuate1(a) # define recordMutableGen_GC(a,b) recordMutableGen(a,b) # define scavenge_loop(a) scavenge_loop1(a) +# define scavenge_block(a) scavenge_block1(a) # define scavenge_mutable_list(bd,g) scavenge_mutable_list1(bd,g) # define scavenge_capability_mut_lists(cap) scavenge_capability_mut_Lists1(cap) #endif @@ -71,6 +73,11 @@ scavengeTSO (StgTSO *tso) debugTrace(DEBUG_gc,"scavenging thread %d",(int)tso->id); + // update the pointer from the Task. + if (tso->bound != NULL) { + tso->bound->tso = tso; + } + saved_eager = gct->eager_promotion; gct->eager_promotion = rtsFalse; @@ -260,7 +267,7 @@ scavenge_srt (StgClosure **srt, nat srt_bitmap) // If the SRT entry hasn't got bit 0 set, the SRT entry points to a // closure that's fixed at link-time, and no extra magic is required. if ( (unsigned long)(*srt) & 0x1 ) { - evacuate(stgCast(StgClosure**,(stgCast(unsigned long, *srt) & ~0x1))); + evacuate( (StgClosure**) ((unsigned long) (*srt) & ~0x1)); } else { evacuate(p); } @@ -299,11 +306,11 @@ scavenge_fun_srt(const StgInfoTable *info) /* ----------------------------------------------------------------------------- Scavenge a block from the given scan pointer up to bd->free. - evac_step is set by the caller to be either zero (for a step in a + evac_gen is set by the caller to be either zero (for a step in a generation < N) or G where G is the generation of the step being scavenged. - We sometimes temporarily change evac_step back to zero if we're + We sometimes temporarily change evac_gen back to zero if we're scavenging a mutable object where eager promotion isn't such a good idea. -------------------------------------------------------------------------- */ @@ -313,20 +320,20 @@ scavenge_block (bdescr *bd) { StgPtr p, q; StgInfoTable *info; - step *saved_evac_step; + generation *saved_evac_gen; rtsBool saved_eager_promotion; - step_workspace *ws; + gen_workspace *ws; - debugTrace(DEBUG_gc, "scavenging block %p (gen %d, step %d) @ %p", - bd->start, bd->gen_no, bd->step->no, bd->u.scan); + debugTrace(DEBUG_gc, "scavenging block %p (gen %d) @ %p", + bd->start, bd->gen_no, bd->u.scan); gct->scan_bd = bd; - gct->evac_step = bd->step; - saved_evac_step = gct->evac_step; + gct->evac_gen = bd->gen; + saved_evac_gen = gct->evac_gen; saved_eager_promotion = gct->eager_promotion; gct->failed_to_evac = rtsFalse; - ws = &gct->steps[bd->step->abs_no]; + ws = &gct->gens[bd->gen->no]; p = bd->u.scan; @@ -604,11 +611,11 @@ scavenge_block (bdescr *bd) case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTVarWatchQueue); break; @@ -617,10 +624,10 @@ scavenge_block (bdescr *bd) case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTVar); break; @@ -629,11 +636,11 @@ scavenge_block (bdescr *bd) case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTRecHeader); break; @@ -644,14 +651,14 @@ scavenge_block (bdescr *bd) StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgTRecChunk); break; @@ -660,10 +667,10 @@ scavenge_block (bdescr *bd) case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgAtomicInvariant); break; @@ -672,11 +679,11 @@ scavenge_block (bdescr *bd) case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable p += sizeofW(StgInvariantCheckQueue); break; @@ -735,14 +742,12 @@ scavenge_mark_stack(void) { StgPtr p, q; StgInfoTable *info; - step *saved_evac_step; + generation *saved_evac_gen; - gct->evac_step = &oldest_gen->steps[0]; - saved_evac_step = gct->evac_step; + gct->evac_gen = oldest_gen; + saved_evac_gen = gct->evac_gen; -linear_scan: - while (!mark_stack_empty()) { - p = pop_mark_stack(); + while ((p = pop_mark_stack())) { ASSERT(LOOKS_LIKE_CLOSURE_PTR(p)); info = get_itbl((StgClosure *)p); @@ -972,11 +977,11 @@ linear_scan: case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -984,10 +989,10 @@ linear_scan: case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -997,14 +1002,14 @@ linear_scan: StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1012,11 +1017,11 @@ linear_scan: case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1024,10 +1029,10 @@ linear_scan: case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1035,11 +1040,11 @@ linear_scan: case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1051,53 +1056,11 @@ linear_scan: if (gct->failed_to_evac) { gct->failed_to_evac = rtsFalse; - if (gct->evac_step) { - recordMutableGen_GC((StgClosure *)q, gct->evac_step->gen_no); + if (gct->evac_gen) { + recordMutableGen_GC((StgClosure *)q, gct->evac_gen->no); } } - - // mark the next bit to indicate "scavenged" - mark(q+1, Bdescr(q)); - - } // while (!mark_stack_empty()) - - // start a new linear scan if the mark stack overflowed at some point - if (mark_stack_overflowed && oldgen_scan_bd == NULL) { - debugTrace(DEBUG_gc, "scavenge_mark_stack: starting linear scan"); - mark_stack_overflowed = rtsFalse; - oldgen_scan_bd = oldest_gen->steps[0].old_blocks; - oldgen_scan = oldgen_scan_bd->start; - } - - if (oldgen_scan_bd) { - // push a new thing on the mark stack - loop: - // find a closure that is marked but not scavenged, and start - // from there. - while (oldgen_scan < oldgen_scan_bd->free - && !is_marked(oldgen_scan,oldgen_scan_bd)) { - oldgen_scan++; - } - - if (oldgen_scan < oldgen_scan_bd->free) { - - // already scavenged? - if (is_marked(oldgen_scan+1,oldgen_scan_bd)) { - oldgen_scan += sizeofW(StgHeader) + MIN_PAYLOAD_SIZE; - goto loop; - } - push_mark_stack(oldgen_scan); - // ToDo: bump the linear scan by the actual size of the object - oldgen_scan += sizeofW(StgHeader) + MIN_PAYLOAD_SIZE; - goto linear_scan; - } - - oldgen_scan_bd = oldgen_scan_bd->link; - if (oldgen_scan_bd != NULL) { - oldgen_scan = oldgen_scan_bd->start; - goto loop; - } - } + } // while (p = pop_mark_stack()) } /* ----------------------------------------------------------------------------- @@ -1112,7 +1075,7 @@ static rtsBool scavenge_one(StgPtr p) { const StgInfoTable *info; - step *saved_evac_step = gct->evac_step; + generation *saved_evac_gen = gct->evac_gen; rtsBool no_luck; ASSERT(LOOKS_LIKE_CLOSURE_PTR(p)); @@ -1289,11 +1252,11 @@ scavenge_one(StgPtr p) case TVAR_WATCH_QUEUE: { StgTVarWatchQueue *wq = ((StgTVarWatchQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&wq->closure); evacuate((StgClosure **)&wq->next_queue_entry); evacuate((StgClosure **)&wq->prev_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1301,10 +1264,10 @@ scavenge_one(StgPtr p) case TVAR: { StgTVar *tvar = ((StgTVar *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tvar->current_value); evacuate((StgClosure **)&tvar->first_watch_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1312,11 +1275,11 @@ scavenge_one(StgPtr p) case TREC_HEADER: { StgTRecHeader *trec = ((StgTRecHeader *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&trec->enclosing_trec); evacuate((StgClosure **)&trec->current_chunk); evacuate((StgClosure **)&trec->invariants_to_check); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1326,14 +1289,14 @@ scavenge_one(StgPtr p) StgWord i; StgTRecChunk *tc = ((StgTRecChunk *) p); TRecEntry *e = &(tc -> entries[0]); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&tc->prev_chunk); for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) { evacuate((StgClosure **)&e->tvar); evacuate((StgClosure **)&e->expected_value); evacuate((StgClosure **)&e->new_value); } - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1341,10 +1304,10 @@ scavenge_one(StgPtr p) case ATOMIC_INVARIANT: { StgAtomicInvariant *invariant = ((StgAtomicInvariant *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate(&invariant->code); evacuate((StgClosure **)&invariant->last_execution); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1352,11 +1315,11 @@ scavenge_one(StgPtr p) case INVARIANT_CHECK_QUEUE: { StgInvariantCheckQueue *queue = ((StgInvariantCheckQueue *) p); - gct->evac_step = 0; + gct->evac_gen = 0; evacuate((StgClosure **)&queue->invariant); evacuate((StgClosure **)&queue->my_execution); evacuate((StgClosure **)&queue->next_queue_entry); - gct->evac_step = saved_evac_step; + gct->evac_gen = saved_evac_gen; gct->failed_to_evac = rtsTrue; // mutable break; } @@ -1376,21 +1339,21 @@ scavenge_one(StgPtr p) * promoted */ { - StgPtr start = gen->steps[0].scan; - bdescr *start_bd = gen->steps[0].scan_bd; + StgPtr start = gen->scan; + bdescr *start_bd = gen->scan_bd; nat size = 0; - scavenge(&gen->steps[0]); - if (start_bd != gen->steps[0].scan_bd) { + scavenge(&gen); + if (start_bd != gen->scan_bd) { size += (P_)BLOCK_ROUND_UP(start) - start; start_bd = start_bd->link; - while (start_bd != gen->steps[0].scan_bd) { + while (start_bd != gen->scan_bd) { size += BLOCK_SIZE_W; start_bd = start_bd->link; } - size += gen->steps[0].scan - - (P_)BLOCK_ROUND_DOWN(gen->steps[0].scan); + size += gen->scan - + (P_)BLOCK_ROUND_DOWN(gen->scan); } else { - size = gen->steps[0].scan - start; + size = gen->scan - start; } debugBelch("evac IND_OLDGEN: %ld bytes", size * sizeof(W_)); } @@ -1419,7 +1382,7 @@ scavenge_mutable_list(bdescr *bd, generation *gen) { StgPtr p, q; - gct->evac_step = &gen->steps[0]; + gct->evac_gen = gen; for (; bd != NULL; bd = bd->link) { for (q = bd->start; q < bd->free; q++) { p = (StgPtr)*q; @@ -1522,7 +1485,7 @@ scavenge_static(void) /* Always evacuate straight to the oldest generation for static * objects */ - gct->evac_step = &oldest_gen->steps[0]; + gct->evac_gen = oldest_gen; /* keep going until we've scavenged all the objects on the linked list... */ @@ -1809,19 +1772,19 @@ scavenge_stack(StgPtr p, StgPtr stack_end) /*----------------------------------------------------------------------------- scavenge the large object list. - evac_step set by caller; similar games played with evac_step as with + evac_gen set by caller; similar games played with evac_gen as with scavenge() - see comment at the top of scavenge(). Most large - objects are (repeatedly) mutable, so most of the time evac_step will + objects are (repeatedly) mutable, so most of the time evac_gen will be zero. --------------------------------------------------------------------------- */ static void -scavenge_large (step_workspace *ws) +scavenge_large (gen_workspace *ws) { bdescr *bd; StgPtr p; - gct->evac_step = ws->step; + gct->evac_gen = ws->gen; bd = ws->todo_large_objects; @@ -1833,15 +1796,15 @@ scavenge_large (step_workspace *ws) // the front when evacuating. ws->todo_large_objects = bd->link; - ACQUIRE_SPIN_LOCK(&ws->step->sync_large_objects); - dbl_link_onto(bd, &ws->step->scavenged_large_objects); - ws->step->n_scavenged_large_blocks += bd->blocks; - RELEASE_SPIN_LOCK(&ws->step->sync_large_objects); + ACQUIRE_SPIN_LOCK(&ws->gen->sync_large_objects); + dbl_link_onto(bd, &ws->gen->scavenged_large_objects); + ws->gen->n_scavenged_large_blocks += bd->blocks; + RELEASE_SPIN_LOCK(&ws->gen->sync_large_objects); p = bd->start; if (scavenge_one(p)) { - if (ws->step->gen_no > 0) { - recordMutableGen_GC((StgClosure *)p, ws->step->gen_no); + if (ws->gen->no > 0) { + recordMutableGen_GC((StgClosure *)p, ws->gen->no); } } @@ -1853,7 +1816,7 @@ scavenge_large (step_workspace *ws) /* ---------------------------------------------------------------------------- Look for work to do. - We look for the oldest step that has either a todo block that can + We look for the oldest gen that has either a todo block that can be scanned, or a block of work on the global queue that we can scan. @@ -1872,8 +1835,8 @@ scavenge_large (step_workspace *ws) static rtsBool scavenge_find_work (void) { - int s; - step_workspace *ws; + int g; + gen_workspace *ws; rtsBool did_something, did_anything; bdescr *bd; @@ -1883,11 +1846,8 @@ scavenge_find_work (void) loop: did_something = rtsFalse; - for (s = total_steps-1; s >= 0; s--) { - if (s == 0 && RtsFlags.GcFlags.generations > 1) { - continue; - } - ws = &gct->steps[s]; + for (g = RtsFlags.GcFlags.generations-1; g >= 0; g--) { + ws = &gct->gens[g]; gct->scan_bd = NULL; @@ -1922,11 +1882,8 @@ loop: #if defined(THREADED_RTS) if (work_stealing) { // look for work to steal - for (s = total_steps-1; s >= 0; s--) { - if (s == 0 && RtsFlags.GcFlags.generations > 1) { - continue; - } - if ((bd = steal_todo_block(s)) != NULL) { + for (g = RtsFlags.GcFlags.generations-1; g >= 0; g--) { + if ((bd = steal_todo_block(g)) != NULL) { scavenge_block(bd); did_something = rtsTrue; break; @@ -1964,8 +1921,7 @@ loop: } // scavenge objects in compacted generation - if (mark_stack_overflowed || oldgen_scan_bd != NULL || - (mark_stack_bdescr != NULL && !mark_stack_empty())) { + if (mark_stack_bd != NULL && !mark_stack_empty()) { scavenge_mark_stack(); work_to_do = rtsTrue; }