X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=rts%2Fsm%2FGC.c;h=c254fcb54fe96f659c2e93dd4e1aee14edf9d565;hb=dc6008a61acedd3d0785111cf8955c479cb226a4;hp=97b32b653d0a890bcd21d0c58dca61b360b578c1;hpb=8ac01a644677a71cbfb4cc5974c3641716d92104;p=ghc-hetmet.git diff --git a/rts/sm/GC.c b/rts/sm/GC.c index 97b32b6..c254fcb 100644 --- a/rts/sm/GC.c +++ b/rts/sm/GC.c @@ -1,6 +1,6 @@ /* ----------------------------------------------------------------------------- * - * (c) The GHC Team 1998-2006 + * (c) The GHC Team 1998-2008 * * Generational garbage collector * @@ -39,16 +39,17 @@ #include "Trace.h" #include "RetainerProfile.h" #include "RaiseAsync.h" -#include "Sparks.h" #include "Papi.h" #include "GC.h" +#include "GCThread.h" #include "Compact.h" #include "Evac.h" #include "Scav.h" #include "GCUtils.h" #include "MarkWeak.h" #include "Sparks.h" +#include "Sweep.h" #include // for memset() #include @@ -128,11 +129,13 @@ long copied; // *words* copied & scavenged during this GC SpinLock recordMutableGen_sync; #endif +DECLARE_GCT + /* ----------------------------------------------------------------------------- Static function declarations -------------------------------------------------------------------------- */ -static void mark_root (StgClosure **root); +static void mark_root (void *user, StgClosure **root); static void zero_static_object_list (StgClosure* first_static); static nat initialise_N (rtsBool force_major_gc); static void alloc_gc_threads (void); @@ -143,7 +146,7 @@ static void update_task_list (void); static void resize_generations (void); static void resize_nursery (void); static void start_gc_threads (void); -static void gc_thread_work (void); +static void scavenge_until_all_done (void); static nat inc_running (void); static nat dec_running (void); static void wakeup_gc_threads (nat n_threads); @@ -181,8 +184,7 @@ GarbageCollect ( rtsBool force_major_gc ) { bdescr *bd; step *stp; - lnat live, allocated, max_copied, avg_copied; - lnat oldgen_saved_blocks = 0; + lnat live, allocated, max_copied, avg_copied, slop; gc_thread *saved_gct; nat g, s, t, n; @@ -202,6 +204,9 @@ GarbageCollect ( rtsBool force_major_gc ) } #endif + ASSERT(sizeof(step_workspace) == 16 * sizeof(StgWord)); + // otherwise adjust the padding in step_workspace. + // tell the stats department that we've started a GC stat_startGC(); @@ -238,10 +243,10 @@ GarbageCollect ( rtsBool force_major_gc ) start_gc_threads(); /* How many threads will be participating in this GC? - * We don't try to parallelise minor GC. + * We don't try to parallelise minor GC, or mark/compact/sweep GC. */ #if defined(THREADED_RTS) - if (n < (4*1024*1024 / BLOCK_SIZE)) { + if (n < (4*1024*1024 / BLOCK_SIZE) || oldest_gen->steps[0].mark) { n_gc_threads = 1; } else { n_gc_threads = RtsFlags.ParFlags.gcThreads; @@ -249,8 +254,8 @@ GarbageCollect ( rtsBool force_major_gc ) #else n_gc_threads = 1; #endif - trace(TRACE_gc|DEBUG_gc, "GC (gen %d): %dKB to collect, using %d thread(s)", - N, n * (BLOCK_SIZE / 1024), n_gc_threads); + trace(TRACE_gc|DEBUG_gc, "GC (gen %d): %d KB to collect, %ld MB in use, using %d thread(s)", + N, n * (BLOCK_SIZE / 1024), mblocks_allocated, n_gc_threads); #ifdef RTS_GTK_FRONTPANEL if (RtsFlags.GcFlags.frontpanel) { @@ -284,10 +289,13 @@ GarbageCollect ( rtsBool force_major_gc ) /* Allocate a mark stack if we're doing a major collection. */ if (major_gc) { - mark_stack_bdescr = allocGroup(MARK_STACK_BLOCKS); + nat mark_stack_blocks; + mark_stack_blocks = stg_max(MARK_STACK_BLOCKS, + oldest_gen->steps[0].n_old_blocks / 100); + mark_stack_bdescr = allocGroup(mark_stack_blocks); mark_stack = (StgPtr *)mark_stack_bdescr->start; mark_sp = mark_stack; - mark_splim = mark_stack + (MARK_STACK_BLOCKS * BLOCK_SIZE_W); + mark_splim = mark_stack + (mark_stack_blocks * BLOCK_SIZE_W); } else { mark_stack_bdescr = NULL; } @@ -322,15 +330,15 @@ GarbageCollect ( rtsBool force_major_gc ) // follow roots from the CAF list (used by GHCi) gct->evac_step = 0; - markCAFs(mark_root); + markCAFs(mark_root, gct); // follow all the roots that the application knows about. gct->evac_step = 0; - GetRoots(mark_root); + markSomeCapabilities(mark_root, gct, gct->thread_index, n_gc_threads); #if defined(RTS_USER_SIGNALS) // mark the signal handlers (signals should be already blocked) - markSignalHandlers(mark_root); + markSignalHandlers(mark_root, gct); #endif // Mark the weak pointer list, and prepare to detect dead weak pointers. @@ -338,7 +346,7 @@ GarbageCollect ( rtsBool force_major_gc ) initWeakForGC(); // Mark the stable pointer table. - markStablePtrTable(mark_root); + markStablePtrTable(mark_root, gct); /* ------------------------------------------------------------------------- * Repeatedly scavenge all the areas we know about until there's no @@ -346,7 +354,7 @@ GarbageCollect ( rtsBool force_major_gc ) */ for (;;) { - gc_thread_work(); + scavenge_until_all_done(); // The other threads are now stopped. We might recurse back to // here, but from now on this is the only thread. @@ -385,14 +393,6 @@ GarbageCollect ( rtsBool force_major_gc ) #endif // NO MORE EVACUATION AFTER THIS POINT! - // Finally: compaction of the oldest generation. - if (major_gc && oldest_gen->steps[0].is_compacted) { - // save number of blocks for stats - oldgen_saved_blocks = oldest_gen->steps[0].n_old_blocks; - compact(); - } - - IF_DEBUG(sanity, checkGlobalTSOList(rtsFalse)); // Two-space collector: free the old to-space. // g0s0->old_blocks is the old nursery @@ -404,9 +404,7 @@ GarbageCollect ( rtsBool force_major_gc ) } } - // For each workspace, in each thread: - // * clear the BF_EVACUATED flag from each copied block - // * move the copied blocks to the step + // For each workspace, in each thread, move the copied blocks to the step { gc_thread *thr; step_workspace *ws; @@ -416,7 +414,12 @@ GarbageCollect ( rtsBool force_major_gc ) thr = gc_threads[t]; // not step 0 - for (s = 1; s < total_steps; s++) { + if (RtsFlags.GcFlags.generations == 1) { + s = 0; + } else { + s = 1; + } + for (; s < total_steps; s++) { ws = &thr->steps[s]; // Push the final block @@ -429,7 +432,6 @@ GarbageCollect ( rtsBool force_major_gc ) prev = NULL; for (bd = ws->scavd_list; bd != NULL; bd = bd->link) { - bd->flags &= ~BF_EVACUATED; // now from-space ws->step->n_words += bd->free - bd->start; prev = bd; } @@ -438,6 +440,23 @@ GarbageCollect ( rtsBool force_major_gc ) ws->step->blocks = ws->scavd_list; } ws->step->n_blocks += ws->n_scavd_blocks; + } + } + + // Add all the partial blocks *after* we've added all the full + // blocks. This is so that we can grab the partial blocks back + // again and try to fill them up in the next GC. + for (t = 0; t < n_gc_threads; t++) { + thr = gc_threads[t]; + + // not step 0 + if (RtsFlags.GcFlags.generations == 1) { + s = 0; + } else { + s = 1; + } + for (; s < total_steps; s++) { + ws = &thr->steps[s]; prev = NULL; for (bd = ws->part_list; bd != NULL; bd = next) { @@ -451,7 +470,6 @@ GarbageCollect ( rtsBool force_major_gc ) freeGroup(bd); ws->n_part_blocks--; } else { - bd->flags &= ~BF_EVACUATED; // now from-space ws->step->n_words += bd->free - bd->start; prev = bd; } @@ -468,19 +486,16 @@ GarbageCollect ( rtsBool force_major_gc ) } } - // Two-space collector: swap the semi-spaces around. - // Currently: g0s0->old_blocks is the old nursery - // g0s0->blocks is to-space from this GC - // We want these the other way around. - if (RtsFlags.GcFlags.generations == 1) { - bdescr *nursery_blocks = g0s0->old_blocks; - nat n_nursery_blocks = g0s0->n_old_blocks; - g0s0->old_blocks = g0s0->blocks; - g0s0->n_old_blocks = g0s0->n_blocks; - g0s0->blocks = nursery_blocks; - g0s0->n_blocks = n_nursery_blocks; + // Finally: compact or sweep the oldest generation. + if (major_gc && oldest_gen->steps[0].mark) { + if (oldest_gen->steps[0].compact) + compact(gct->scavenged_static_objects); + else + sweep(&oldest_gen->steps[0]); } + IF_DEBUG(sanity, checkGlobalTSOList(rtsFalse)); + /* run through all the generations/steps and tidy up */ copied = 0; @@ -531,7 +546,7 @@ GarbageCollect ( rtsBool force_major_gc ) } for (s = 0; s < generations[g].n_steps; s++) { - bdescr *next; + bdescr *next, *prev; stp = &generations[g].steps[s]; // for generations we collected... @@ -542,28 +557,48 @@ GarbageCollect ( rtsBool force_major_gc ) * freed blocks will probaby be quickly recycled. */ if (!(g == 0 && s == 0 && RtsFlags.GcFlags.generations > 1)) { - if (stp->is_compacted) + if (stp->mark) { - // for a compacted step, just shift the new to-space - // onto the front of the now-compacted existing blocks. - for (bd = stp->blocks; bd != NULL; bd = bd->link) { - bd->flags &= ~BF_EVACUATED; // now from-space - stp->n_words += bd->free - bd->start; - } // tack the new blocks on the end of the existing blocks if (stp->old_blocks != NULL) { + + prev = NULL; for (bd = stp->old_blocks; bd != NULL; bd = next) { - // NB. this step might not be compacted next - // time, so reset the BF_COMPACTED flags. - // They are set before GC if we're going to - // compact. (search for BF_COMPACTED above). - bd->flags &= ~BF_COMPACTED; - next = bd->link; - if (next == NULL) { - bd->link = stp->blocks; - } + + next = bd->link; + + if (!(bd->flags & BF_MARKED)) + { + if (prev == NULL) { + stp->old_blocks = next; + } else { + prev->link = next; + } + freeGroup(bd); + stp->n_old_blocks--; + } + else + { + stp->n_words += bd->free - bd->start; + + // NB. this step might not be compacted next + // time, so reset the BF_MARKED flags. + // They are set before GC if we're going to + // compact. (search for BF_MARKED above). + bd->flags &= ~BF_MARKED; + + // between GCs, all blocks in the heap except + // for the nursery have the BF_EVACUATED flag set. + bd->flags |= BF_EVACUATED; + + prev = bd; + } } - stp->blocks = stp->old_blocks; + + if (prev != NULL) { + prev->link = stp->blocks; + stp->blocks = stp->old_blocks; + } } // add the new blocks to the block tally stp->n_blocks += stp->n_old_blocks; @@ -589,10 +624,6 @@ GarbageCollect ( rtsBool force_major_gc ) bd = next; } - // update the count of blocks used by large objects - for (bd = stp->scavenged_large_objects; bd != NULL; bd = bd->link) { - bd->flags &= ~BF_EVACUATED; - } stp->large_objects = stp->scavenged_large_objects; stp->n_large_blocks = stp->n_scavenged_large_blocks; @@ -605,7 +636,6 @@ GarbageCollect ( rtsBool force_major_gc ) */ for (bd = stp->scavenged_large_objects; bd; bd = next) { next = bd->link; - bd->flags &= ~BF_EVACUATED; dbl_link_onto(bd, &stp->large_objects); } @@ -689,6 +719,7 @@ GarbageCollect ( rtsBool force_major_gc ) // send exceptions to any threads which were about to die RELEASE_SM_LOCK; resurrectThreads(resurrected_threads); + performPendingThrowTos(exception_threads); ACQUIRE_SM_LOCK; // Update the stable pointer hash table. @@ -722,7 +753,8 @@ GarbageCollect ( rtsBool force_major_gc ) #endif // ok, GC over: tell the stats department what happened. - stat_endGC(allocated, live, copied, N, max_copied, avg_copied); + slop = calcLiveBlocks() * BLOCK_SIZE_W - live; + stat_endGC(allocated, live, copied, N, max_copied, avg_copied, slop); #if defined(RTS_USER_SIGNALS) if (RtsFlags.MiscFlags.install_signal_handlers) { @@ -737,212 +769,6 @@ GarbageCollect ( rtsBool force_major_gc ) } /* ----------------------------------------------------------------------------- - * Mark all nodes pointed to by sparks in the spark queues (for GC) Does an - * implicit slide i.e. after marking all sparks are at the beginning of the - * spark pool and the spark pool only contains sparkable closures - * -------------------------------------------------------------------------- */ - -#ifdef THREADED_RTS -static void -markSparkQueue (evac_fn evac, Capability *cap) -{ - StgClosure **sparkp, **to_sparkp; - nat n, pruned_sparks; // stats only - StgSparkPool *pool; - - PAR_TICKY_MARK_SPARK_QUEUE_START(); - - n = 0; - pruned_sparks = 0; - - pool = &(cap->r.rSparks); - - ASSERT_SPARK_POOL_INVARIANTS(pool); - -#if defined(PARALLEL_HASKELL) - // stats only - n = 0; - pruned_sparks = 0; -#endif - - sparkp = pool->hd; - to_sparkp = pool->hd; - while (sparkp != pool->tl) { - ASSERT(*sparkp!=NULL); - ASSERT(LOOKS_LIKE_CLOSURE_PTR(((StgClosure *)*sparkp))); - // ToDo?: statistics gathering here (also for GUM!) - if (closure_SHOULD_SPARK(*sparkp)) { - evac(sparkp); - *to_sparkp++ = *sparkp; - if (to_sparkp == pool->lim) { - to_sparkp = pool->base; - } - n++; - } else { - pruned_sparks++; - } - sparkp++; - if (sparkp == pool->lim) { - sparkp = pool->base; - } - } - pool->tl = to_sparkp; - - PAR_TICKY_MARK_SPARK_QUEUE_END(n); - -#if defined(PARALLEL_HASKELL) - debugTrace(DEBUG_sched, - "marked %d sparks and pruned %d sparks on [%x]", - n, pruned_sparks, mytid); -#else - debugTrace(DEBUG_sched, - "marked %d sparks and pruned %d sparks", - n, pruned_sparks); -#endif - - debugTrace(DEBUG_sched, - "new spark queue len=%d; (hd=%p; tl=%p)\n", - sparkPoolSize(pool), pool->hd, pool->tl); -} -#endif - -/* --------------------------------------------------------------------------- - Where are the roots that we know about? - - - all the threads on the runnable queue - - all the threads on the blocked queue - - all the threads on the sleeping queue - - all the thread currently executing a _ccall_GC - - all the "main threads" - - ------------------------------------------------------------------------ */ - -void -GetRoots( evac_fn evac ) -{ - nat i; - Capability *cap; - Task *task; - - // Each GC thread is responsible for following roots from the - // Capability of the same number. There will usually be the same - // or fewer Capabilities as GC threads, but just in case there - // are more, we mark every Capability whose number is the GC - // thread's index plus a multiple of the number of GC threads. - for (i = gct->thread_index; i < n_capabilities; i += n_gc_threads) { - cap = &capabilities[i]; - evac((StgClosure **)(void *)&cap->run_queue_hd); - evac((StgClosure **)(void *)&cap->run_queue_tl); -#if defined(THREADED_RTS) - evac((StgClosure **)(void *)&cap->wakeup_queue_hd); - evac((StgClosure **)(void *)&cap->wakeup_queue_tl); -#endif - for (task = cap->suspended_ccalling_tasks; task != NULL; - task=task->next) { - debugTrace(DEBUG_sched, - "evac'ing suspended TSO %lu", (unsigned long)task->suspended_tso->id); - evac((StgClosure **)(void *)&task->suspended_tso); - } - -#if defined(THREADED_RTS) - markSparkQueue(evac,cap); -#endif - } - -#if !defined(THREADED_RTS) - evac((StgClosure **)(void *)&blocked_queue_hd); - evac((StgClosure **)(void *)&blocked_queue_tl); - evac((StgClosure **)(void *)&sleeping_queue); -#endif -} - -/* ----------------------------------------------------------------------------- - isAlive determines whether the given closure is still alive (after - a garbage collection) or not. It returns the new address of the - closure if it is alive, or NULL otherwise. - - NOTE: Use it before compaction only! - It untags and (if needed) retags pointers to closures. - -------------------------------------------------------------------------- */ - - -StgClosure * -isAlive(StgClosure *p) -{ - const StgInfoTable *info; - bdescr *bd; - StgWord tag; - StgClosure *q; - - while (1) { - /* The tag and the pointer are split, to be merged later when needed. */ - tag = GET_CLOSURE_TAG(p); - q = UNTAG_CLOSURE(p); - - ASSERT(LOOKS_LIKE_CLOSURE_PTR(q)); - info = get_itbl(q); - - // ignore static closures - // - // ToDo: for static closures, check the static link field. - // Problem here is that we sometimes don't set the link field, eg. - // for static closures with an empty SRT or CONSTR_STATIC_NOCAFs. - // - if (!HEAP_ALLOCED(q)) { - return p; - } - - // ignore closures in generations that we're not collecting. - bd = Bdescr((P_)q); - if (bd->gen_no > N) { - return p; - } - - // if it's a pointer into to-space, then we're done - if (bd->flags & BF_EVACUATED) { - return p; - } - - // large objects use the evacuated flag - if (bd->flags & BF_LARGE) { - return NULL; - } - - // check the mark bit for compacted steps - if ((bd->flags & BF_COMPACTED) && is_marked((P_)q,bd)) { - return p; - } - - switch (info->type) { - - case IND: - case IND_STATIC: - case IND_PERM: - case IND_OLDGEN: // rely on compatible layout with StgInd - case IND_OLDGEN_PERM: - // follow indirections - p = ((StgInd *)q)->indirectee; - continue; - - case EVACUATED: - // alive! - return ((StgEvacuated *)q)->evacuee; - - case TSO: - if (((StgTSO *)q)->what_next == ThreadRelocated) { - p = (StgClosure *)((StgTSO *)q)->link; - continue; - } - return NULL; - - default: - // dead. - return NULL; - } - } -} - -/* ----------------------------------------------------------------------------- Figure out which generation to collect, initialise N and major_gc. Also returns the total number of blocks in generations that will be @@ -1092,28 +918,57 @@ dec_running (void) return n_running; } -// -// gc_thread_work(): Scavenge until there's no work left to do and all -// the running threads are idle. -// +static rtsBool +any_work (void) +{ + int s; + step_workspace *ws; + + gct->any_work++; + + write_barrier(); + + // scavenge objects in compacted generation + if (mark_stack_overflowed || oldgen_scan_bd != NULL || + (mark_stack_bdescr != NULL && !mark_stack_empty())) { + return rtsTrue; + } + + // Check for global work in any step. We don't need to check for + // local work, because we have already exited scavenge_loop(), + // which means there is no local work for this thread. + for (s = total_steps-1; s >= 0; s--) { + if (s == 0 && RtsFlags.GcFlags.generations > 1) { + continue; + } + ws = &gct->steps[s]; + if (ws->todo_large_objects) return rtsTrue; + if (ws->step->todos) return rtsTrue; + } + + gct->no_work++; + + return rtsFalse; +} + static void -gc_thread_work (void) +scavenge_until_all_done (void) { nat r; debugTrace(DEBUG_gc, "GC thread %d working", gct->thread_index); - // gc_running_threads has already been incremented for us; either - // this is the main thread and we incremented it inside - // GarbageCollect(), or this is a worker thread and the main - // thread bumped gc_running_threads before waking us up. - - // Every thread evacuates some roots. - gct->evac_step = 0; - GetRoots(mark_root); - loop: +#if defined(THREADED_RTS) + if (n_gc_threads > 1) { + scavenge_loop(); + } else { + scavenge_loop1(); + } +#else scavenge_loop(); +#endif + // scavenge_loop() only exits when there's no work to do r = dec_running(); @@ -1121,7 +976,7 @@ loop: gct->thread_index, r); while (gc_running_threads != 0) { - usleep(1); + // usleep(1); if (any_work()) { inc_running(); goto loop; @@ -1136,8 +991,26 @@ loop: debugTrace(DEBUG_gc, "GC thread %d finished.", gct->thread_index); } - #if defined(THREADED_RTS) +// +// gc_thread_work(): Scavenge until there's no work left to do and all +// the running threads are idle. +// +static void +gc_thread_work (void) +{ + // gc_running_threads has already been incremented for us; this is + // a worker thread and the main thread bumped gc_running_threads + // before waking us up. + + // Every thread evacuates some roots. + gct->evac_step = 0; + markSomeCapabilities(mark_root, gct, gct->thread_index, n_gc_threads); + + scavenge_until_all_done(); +} + + static void gc_thread_mainloop (void) { @@ -1275,20 +1148,26 @@ init_collected_gen (nat g, nat n_threads) for (s = 0; s < generations[g].n_steps; s++) { + stp = &generations[g].steps[s]; + ASSERT(stp->gen_no == g); + + // we'll construct a new list of threads in this step + // during GC, throw away the current list. + stp->old_threads = stp->threads; + stp->threads = END_TSO_QUEUE; + // generation 0, step 0 doesn't need to-space if (g == 0 && s == 0 && RtsFlags.GcFlags.generations > 1) { continue; } - stp = &generations[g].steps[s]; - ASSERT(stp->gen_no == g); - // deprecate the existing blocks stp->old_blocks = stp->blocks; stp->n_old_blocks = stp->n_blocks; stp->blocks = NULL; stp->n_blocks = 0; stp->n_words = 0; + stp->live_estimate = 0; // we don't have any to-be-scavenged blocks yet stp->todos = NULL; @@ -1299,13 +1178,18 @@ init_collected_gen (nat g, nat n_threads) stp->scavenged_large_objects = NULL; stp->n_scavenged_large_blocks = 0; - // mark the large objects as not evacuated yet + // mark the small objects as from-space + for (bd = stp->old_blocks; bd; bd = bd->link) { + bd->flags &= ~BF_EVACUATED; + } + + // mark the large objects as from-space for (bd = stp->large_objects; bd; bd = bd->link) { bd->flags &= ~BF_EVACUATED; } // for a compacted step, we need to allocate the bitmap - if (stp->is_compacted) { + if (stp->mark) { nat bitmap_size; // in bytes bdescr *bitmap_bdescr; StgWord *bitmap; @@ -1330,12 +1214,14 @@ init_collected_gen (nat g, nat n_threads) bd->u.bitmap = bitmap; bitmap += BLOCK_SIZE_W / (sizeof(W_)*BITS_PER_BYTE); - // Also at this point we set the BF_COMPACTED flag + // Also at this point we set the BF_MARKED flag // for this block. The invariant is that - // BF_COMPACTED is always unset, except during GC + // BF_MARKED is always unset, except during GC // when it is set on those blocks which will be // compacted. - bd->flags |= BF_COMPACTED; + if (!(bd->flags & BF_FRAGMENTED)) { + bd->flags |= BF_MARKED; + } } } } @@ -1388,11 +1274,12 @@ init_uncollected_gen (nat g, nat threads) stp->n_scavenged_large_blocks = 0; } - for (t = 0; t < threads; t++) { - for (s = 0; s < generations[g].n_steps; s++) { + for (s = 0; s < generations[g].n_steps; s++) { + stp = &generations[g].steps[s]; + + for (t = 0; t < threads; t++) { ws = &gc_threads[t]->steps[g * RtsFlags.GcFlags.steps + s]; - stp = ws->step; ws->buffer_todo_bd = NULL; ws->todo_large_objects = NULL; @@ -1423,8 +1310,26 @@ init_uncollected_gen (nat g, nat threads) alloc_todo_block(ws,0); } } + + // deal out any more partial blocks to the threads' part_lists + t = 0; + while (stp->blocks && isPartiallyFull(stp->blocks)) + { + bd = stp->blocks; + stp->blocks = bd->link; + ws = &gc_threads[t]->steps[g * RtsFlags.GcFlags.steps + s]; + bd->link = ws->part_list; + ws->part_list = bd; + ws->n_part_blocks += 1; + bd->u.scan = bd->free; + stp->n_blocks -= 1; + stp->n_words -= bd->free - bd->start; + t++; + if (t == n_gc_threads) t = 0; + } } + // Move the private mutable lists from each capability onto the // main mutable list for the generation. for (i = 0; i < n_capabilities; i++) { @@ -1460,13 +1365,24 @@ init_gc_thread (gc_thread *t) } /* ----------------------------------------------------------------------------- - Function we pass to GetRoots to evacuate roots. + Function we pass to evacuate roots. -------------------------------------------------------------------------- */ static void -mark_root(StgClosure **root) +mark_root(void *user, StgClosure **root) { - evacuate(root); + // we stole a register for gct, but this function is called from + // *outside* the GC where the register variable is not in effect, + // so we need to save and restore it here. NB. only call + // mark_root() from the main GC thread, otherwise gct will be + // incorrect. + gc_thread *saved_gct; + saved_gct = gct; + gct = user; + + evacuate(root); + + gct = saved_gct; } /* ----------------------------------------------------------------------------- @@ -1487,42 +1403,6 @@ zero_static_object_list(StgClosure* first_static) } } -/* ----------------------------------------------------------------------------- - Reverting CAFs - -------------------------------------------------------------------------- */ - -void -revertCAFs( void ) -{ - StgIndStatic *c; - - for (c = (StgIndStatic *)revertible_caf_list; c != NULL; - c = (StgIndStatic *)c->static_link) - { - SET_INFO(c, c->saved_info); - c->saved_info = NULL; - // could, but not necessary: c->static_link = NULL; - } - revertible_caf_list = NULL; -} - -void -markCAFs( evac_fn evac ) -{ - StgIndStatic *c; - - for (c = (StgIndStatic *)caf_list; c != NULL; - c = (StgIndStatic *)c->static_link) - { - evac(&c->indirectee); - } - for (c = (StgIndStatic *)revertible_caf_list; c != NULL; - c = (StgIndStatic *)c->static_link) - { - evac(&c->indirectee); - } -} - /* ---------------------------------------------------------------------------- Update the pointers from the task list @@ -1571,13 +1451,18 @@ resize_generations (void) nat g; if (major_gc && RtsFlags.GcFlags.generations > 1) { - nat live, size, min_alloc; + nat live, size, min_alloc, words; nat max = RtsFlags.GcFlags.maxHeapSize; nat gens = RtsFlags.GcFlags.generations; // live in the oldest generations - live = (oldest_gen->steps[0].n_words + BLOCK_SIZE_W - 1) / BLOCK_SIZE_W+ - oldest_gen->steps[0].n_large_blocks; + if (oldest_gen->steps[0].live_estimate != 0) { + words = oldest_gen->steps[0].live_estimate; + } else { + words = oldest_gen->steps[0].n_words; + } + live = (words + BLOCK_SIZE_W - 1) / BLOCK_SIZE_W + + oldest_gen->steps[0].n_large_blocks; // default max size for all generations except zero size = stg_max(live * RtsFlags.GcFlags.oldGenFactor, @@ -1594,13 +1479,19 @@ resize_generations (void) (max > 0 && oldest_gen->steps[0].n_blocks > (RtsFlags.GcFlags.compactThreshold * max) / 100))) { - oldest_gen->steps[0].is_compacted = 1; + oldest_gen->steps[0].mark = 1; + oldest_gen->steps[0].compact = 1; // debugBelch("compaction: on\n", live); } else { - oldest_gen->steps[0].is_compacted = 0; + oldest_gen->steps[0].mark = 0; + oldest_gen->steps[0].compact = 0; // debugBelch("compaction: off\n", live); } + if (RtsFlags.GcFlags.sweep) { + oldest_gen->steps[0].mark = 1; + } + // if we're going to go over the maximum heap size, reduce the // size of the generations accordingly. The calculation is // different if compaction is turned on, because we don't need @@ -1614,7 +1505,7 @@ resize_generations (void) heapOverflow(); } - if (oldest_gen->steps[0].is_compacted) { + if (oldest_gen->steps[0].compact) { if ( (size + (size - 1) * (gens - 2) * 2) + min_alloc > max ) { size = (max - min_alloc) / ((gens - 1) * 2 - 1); } @@ -1665,7 +1556,7 @@ resize_nursery (void) * performance we get from 3L bytes, reducing to the same * performance at 2L bytes. */ - blocks = g0s0->n_old_blocks; + blocks = g0s0->n_blocks; if ( RtsFlags.GcFlags.maxHeapSize != 0 && blocks * RtsFlags.GcFlags.oldGenFactor * 2 >