X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=rts%2Fsm%2FGC.c;h=07cf2b51e5f3fa92b51f74bff1a170fb79b2a41d;hb=754e039a8a15d5774fe73872ff9ac593b46370e0;hp=501a9e52d4ba0f6062d9472e0093a601fa63c6fb;hpb=54075ea5872c151573710d32876314e2005984ba;p=ghc-hetmet.git diff --git a/rts/sm/GC.c b/rts/sm/GC.c index 501a9e5..07cf2b5 100644 --- a/rts/sm/GC.c +++ b/rts/sm/GC.c @@ -11,35 +11,32 @@ * * ---------------------------------------------------------------------------*/ -// #include "PosixSource.h" +#include "PosixSource.h" #include "Rts.h" -#include "RtsFlags.h" +#include "HsFFI.h" + +#include "Storage.h" #include "RtsUtils.h" #include "Apply.h" -#include "OSThreads.h" -#include "LdvProfile.h" #include "Updates.h" #include "Stats.h" #include "Schedule.h" #include "Sanity.h" #include "BlockAlloc.h" -#include "MBlock.h" #include "ProfHeap.h" -#include "SchedAPI.h" #include "Weak.h" #include "Prelude.h" -#include "ParTicky.h" // ToDo: move into Rts.h #include "RtsSignals.h" #include "STM.h" -#include "HsFFI.h" -#include "Linker.h" #if defined(RTS_GTK_FRONTPANEL) #include "FrontPanel.h" #endif #include "Trace.h" #include "RetainerProfile.h" +#include "LdvProfile.h" #include "RaiseAsync.h" #include "Papi.h" +#include "Stable.h" #include "GC.h" #include "GCThread.h" @@ -47,6 +44,7 @@ #include "Evac.h" #include "Scav.h" #include "GCUtils.h" +#include "MarkStack.h" #include "MarkWeak.h" #include "Sparks.h" #include "Sweep.h" @@ -116,7 +114,10 @@ nat mutlist_MUTVARS, /* Thread-local data for each GC thread */ gc_thread **gc_threads = NULL; -// gc_thread *gct = NULL; // this thread's gct TODO: make thread-local + +#if !defined(THREADED_RTS) +StgWord8 the_gc_thread[sizeof(gc_thread) + 64 * sizeof(step_workspace)]; +#endif // Number of threads running in *this* GC. Affects how many // step->todos[] lists we have to look in to find work. @@ -144,8 +145,8 @@ static void resize_generations (void); static void resize_nursery (void); static void start_gc_threads (void); static void scavenge_until_all_done (void); -static nat inc_running (void); -static nat dec_running (void); +static StgWord inc_running (void); +static StgWord dec_running (void); static void wakeup_gc_threads (nat n_threads, nat me); static void shutdown_gc_threads (nat n_threads, nat me); @@ -154,21 +155,12 @@ static void gcCAFs (void); #endif /* ----------------------------------------------------------------------------- - The mark bitmap & stack. + The mark stack. -------------------------------------------------------------------------- */ -#define MARK_STACK_BLOCKS 4 - -bdescr *mark_stack_bdescr; -StgPtr *mark_stack; -StgPtr *mark_sp; -StgPtr *mark_splim; - -// Flag and pointers used for falling back to a linear scan when the -// mark stack overflows. -rtsBool mark_stack_overflowed; -bdescr *oldgen_scan_bd; -StgPtr oldgen_scan; +bdescr *mark_stack_top_bd; // topmost block in the mark stack +bdescr *mark_stack_bd; // current block in the mark stack +StgPtr mark_sp; // pointer to the next unallocated mark stack entry /* ----------------------------------------------------------------------------- GarbageCollect: the main entry point to the garbage collector. @@ -212,6 +204,9 @@ GarbageCollect (rtsBool force_major_gc, // tell the STM to discard any cached closures it's hoping to re-use stmPreGCHook(); + // lock the StablePtr table + stablePtrPreGC(); + #ifdef DEBUG mutlist_MUTVARS = 0; mutlist_MUTARRS = 0; @@ -234,7 +229,8 @@ GarbageCollect (rtsBool force_major_gc, n = initialise_N(force_major_gc); #if defined(THREADED_RTS) - work_stealing = RtsFlags.ParFlags.parGcLoadBalancing; + work_stealing = RtsFlags.ParFlags.parGcLoadBalancingEnabled && + N >= RtsFlags.ParFlags.parGcLoadBalancingGen; // It's not always a good idea to do load balancing in parallel // GC. In particular, for a parallel program we don't want to // lose locality by moving cached data into another CPU's cache @@ -264,7 +260,7 @@ GarbageCollect (rtsBool force_major_gc, n_gc_threads = 1; #endif - trace(TRACE_gc|DEBUG_gc, "GC (gen %d): %d KB to collect, %ld MB in use, using %d thread(s)", + debugTrace(DEBUG_gc, "GC (gen %d): %d KB to collect, %ld MB in use, using %d thread(s)", N, n * (BLOCK_SIZE / 1024), mblocks_allocated, n_gc_threads); #ifdef RTS_GTK_FRONTPANEL @@ -275,7 +271,7 @@ GarbageCollect (rtsBool force_major_gc, #ifdef DEBUG // check for memory leaks if DEBUG is on - memInventory(traceClass(DEBUG_gc)); + memInventory(DEBUG_gc); #endif // check stack sanity *before* GC @@ -300,26 +296,26 @@ GarbageCollect (rtsBool force_major_gc, /* Allocate a mark stack if we're doing a major collection. */ if (major_gc && oldest_gen->steps[0].mark) { - nat mark_stack_blocks; - mark_stack_blocks = stg_max(MARK_STACK_BLOCKS, - oldest_gen->steps[0].n_old_blocks / 100); - mark_stack_bdescr = allocGroup(mark_stack_blocks); - mark_stack = (StgPtr *)mark_stack_bdescr->start; - mark_sp = mark_stack; - mark_splim = mark_stack + (mark_stack_blocks * BLOCK_SIZE_W); + mark_stack_bd = allocBlock(); + mark_stack_top_bd = mark_stack_bd; + mark_stack_bd->link = NULL; + mark_stack_bd->u.back = NULL; + mark_sp = mark_stack_bd->start; } else { - mark_stack_bdescr = NULL; + mark_stack_bd = NULL; + mark_stack_top_bd = NULL; + mark_sp = NULL; } // this is the main thread #ifdef THREADED_RTS if (n_gc_threads == 1) { - gct = gc_threads[0]; + SET_GCT(gc_threads[0]); } else { - gct = gc_threads[cap->no]; + SET_GCT(gc_threads[cap->no]); } #else - gct = gc_threads[0]; +SET_GCT(gc_threads[0]); #endif /* ----------------------------------------------------------------------- @@ -534,12 +530,12 @@ GarbageCollect (rtsBool force_major_gc, nat i; for (i=0; i < n_gc_threads; i++) { if (n_gc_threads > 1) { - trace(TRACE_gc,"thread %d:", i); - trace(TRACE_gc," copied %ld", gc_threads[i]->copied * sizeof(W_)); - trace(TRACE_gc," scanned %ld", gc_threads[i]->scanned * sizeof(W_)); - trace(TRACE_gc," any_work %ld", gc_threads[i]->any_work); - trace(TRACE_gc," no_work %ld", gc_threads[i]->no_work); - trace(TRACE_gc," scav_find_work %ld", gc_threads[i]->scav_find_work); + debugTrace(DEBUG_gc,"thread %d:", i); + debugTrace(DEBUG_gc," copied %ld", gc_threads[i]->copied * sizeof(W_)); + debugTrace(DEBUG_gc," scanned %ld", gc_threads[i]->scanned * sizeof(W_)); + debugTrace(DEBUG_gc," any_work %ld", gc_threads[i]->any_work); + debugTrace(DEBUG_gc," no_work %ld", gc_threads[i]->no_work); + debugTrace(DEBUG_gc," scav_find_work %ld", gc_threads[i]->scav_find_work); } copied += gc_threads[i]->copied; max_copied = stg_max(gc_threads[i]->copied, max_copied); @@ -703,8 +699,10 @@ GarbageCollect (rtsBool force_major_gc, pinned_object_block = NULL; // Free the mark stack. - if (mark_stack_bdescr != NULL) { - freeGroup(mark_stack_bdescr); + if (mark_stack_top_bd != NULL) { + debugTrace(DEBUG_gc, "mark stack: %d blocks", + countBlocks(mark_stack_top_bd)); + freeChain(mark_stack_top_bd); } // Free any bitmaps. @@ -764,7 +762,7 @@ GarbageCollect (rtsBool force_major_gc, IF_DEBUG(sanity, checkSanity()); // extra GC trace info - if (traceClass(TRACE_gc|DEBUG_gc)) statDescribeGens(); + IF_DEBUG(gc, statDescribeGens()); #ifdef DEBUG // symbol-table based profiling @@ -778,7 +776,7 @@ GarbageCollect (rtsBool force_major_gc, #ifdef DEBUG // check for memory leaks if DEBUG is on - memInventory(traceClass(DEBUG_gc)); + memInventory(DEBUG_gc); #endif #ifdef RTS_GTK_FRONTPANEL @@ -791,6 +789,9 @@ GarbageCollect (rtsBool force_major_gc, slop = calcLiveBlocks() * BLOCK_SIZE_W - live; stat_endGC(allocated, live, copied, N, max_copied, avg_copied, slop); + // unlock the StablePtr table + stablePtrPostGC(); + // Guess which generation we'll collect *next* time initialise_N(force_major_gc); @@ -803,7 +804,7 @@ GarbageCollect (rtsBool force_major_gc, RELEASE_SM_LOCK; - gct = saved_gct; + SET_GCT(saved_gct); } /* ----------------------------------------------------------------------------- @@ -857,15 +858,11 @@ initialise_N (rtsBool force_major_gc) #define GC_THREAD_RUNNING 2 #define GC_THREAD_WAITING_TO_CONTINUE 3 -static gc_thread * -alloc_gc_thread (int n) +static void +new_gc_thread (nat n, gc_thread *t) { nat s; step_workspace *ws; - gc_thread *t; - - t = stgMallocBytes(sizeof(gc_thread) + total_steps * sizeof(step_workspace), - "alloc_gc_thread"); #ifdef THREADED_RTS t->id = 0; @@ -891,7 +888,7 @@ alloc_gc_thread (int n) ws = &t->steps[s]; ws->step = &all_steps[s]; ASSERT(s == ws->step->abs_no); - ws->gct = t; + ws->my_gct = t; ws->todo_bd = NULL; ws->todo_q = newWSDeque(128); @@ -904,8 +901,6 @@ alloc_gc_thread (int n) ws->scavd_list = NULL; ws->n_scavd_blocks = 0; } - - return t; } @@ -920,14 +915,34 @@ initGcThreads (void) "alloc_gc_threads"); for (i = 0; i < RtsFlags.ParFlags.nNodes; i++) { - gc_threads[i] = alloc_gc_thread(i); + gc_threads[i] = + stgMallocBytes(sizeof(gc_thread) + total_steps * sizeof(step_workspace), + "alloc_gc_threads"); + + new_gc_thread(i, gc_threads[i]); } #else - gc_threads = stgMallocBytes (sizeof(gc_thread*), - "alloc_gc_threads"); + gc_threads = stgMallocBytes (sizeof(gc_thread*),"alloc_gc_threads"); + gc_threads[0] = gct; + new_gc_thread(0,gc_threads[0]); +#endif + } +} - gc_threads[0] = alloc_gc_thread(0); +void +freeGcThreads (void) +{ + if (gc_threads != NULL) { +#if defined(THREADED_RTS) + nat i; + for (i = 0; i < RtsFlags.ParFlags.nNodes; i++) { + stgFree (gc_threads[i]); + } + stgFree (gc_threads); +#else + stgFree (gc_threads); #endif + gc_threads = NULL; } } @@ -935,32 +950,22 @@ initGcThreads (void) Start GC threads ------------------------------------------------------------------------- */ -static nat gc_running_threads; - -#if defined(THREADED_RTS) -static Mutex gc_running_mutex; -#endif +static volatile StgWord gc_running_threads; -static nat +static StgWord inc_running (void) { - nat n_running; - ACQUIRE_LOCK(&gc_running_mutex); - n_running = ++gc_running_threads; - RELEASE_LOCK(&gc_running_mutex); - ASSERT(n_running <= n_gc_threads); - return n_running; + StgWord new; + new = atomic_inc(&gc_running_threads); + ASSERT(new <= n_gc_threads); + return new; } -static nat +static StgWord dec_running (void) { - nat n_running; - ACQUIRE_LOCK(&gc_running_mutex); - ASSERT(n_gc_threads != 0); - n_running = --gc_running_threads; - RELEASE_LOCK(&gc_running_mutex); - return n_running; + ASSERT(gc_running_threads != 0); + return atomic_dec(&gc_running_threads); } static rtsBool @@ -974,8 +979,7 @@ any_work (void) write_barrier(); // scavenge objects in compacted generation - if (mark_stack_overflowed || oldgen_scan_bd != NULL || - (mark_stack_bdescr != NULL && !mark_stack_empty())) { + if (mark_stack_bd != NULL && !mark_stack_empty()) { return rtsTrue; } @@ -1056,6 +1060,11 @@ loop: void gcWorkerThread (Capability *cap) { + gc_thread *saved_gct; + + // necessary if we stole a callee-saves register for gct: + saved_gct = gct; + cap->in_gc = rtsTrue; gct = gc_threads[cap->no]; @@ -1095,14 +1104,17 @@ gcWorkerThread (Capability *cap) gct->thread_index); ACQUIRE_SPIN_LOCK(&gct->mut_spin); debugTrace(DEBUG_gc, "GC thread %d on my way...", gct->thread_index); + + SET_GCT(saved_gct); } #endif +#if defined(THREADED_RTS) + void waitForGcThreads (Capability *cap USED_IF_THREADS) { -#if defined(THREADED_RTS) nat n_threads = RtsFlags.ParFlags.nNodes; nat me = cap->no; nat i, j; @@ -1115,7 +1127,7 @@ waitForGcThreads (Capability *cap USED_IF_THREADS) prodCapability(&capabilities[i], cap->running_task); } } - for (j=0; j < 10000000; j++) { + for (j=0; j < 10; j++) { retry = rtsFalse; for (i=0; i < n_threads; i++) { if (i == me) continue; @@ -1126,17 +1138,18 @@ waitForGcThreads (Capability *cap USED_IF_THREADS) } } if (!retry) break; + yieldThread(); } } -#endif } +#endif // THREADED_RTS + static void start_gc_threads (void) { #if defined(THREADED_RTS) gc_running_threads = 0; - initMutex(&gc_running_mutex); #endif } @@ -1173,10 +1186,10 @@ shutdown_gc_threads (nat n_threads USED_IF_THREADS, nat me USED_IF_THREADS) #endif } +#if defined(THREADED_RTS) void releaseGCThreads (Capability *cap USED_IF_THREADS) { -#if defined(THREADED_RTS) nat n_threads = RtsFlags.ParFlags.nNodes; nat me = cap->no; nat i; @@ -1189,8 +1202,8 @@ releaseGCThreads (Capability *cap USED_IF_THREADS) ACQUIRE_SPIN_LOCK(&gc_threads[i]->gc_spin); RELEASE_SPIN_LOCK(&gc_threads[i]->mut_spin); } -#endif } +#endif /* ---------------------------------------------------------------------------- Initialise a generation that is to be collected @@ -1435,7 +1448,7 @@ init_gc_thread (gc_thread *t) -------------------------------------------------------------------------- */ static void -mark_root(void *user, StgClosure **root) +mark_root(void *user USED_IF_THREADS, StgClosure **root) { // we stole a register for gct, but this function is called from // *outside* the GC where the register variable is not in effect, @@ -1444,11 +1457,11 @@ mark_root(void *user, StgClosure **root) // incorrect. gc_thread *saved_gct; saved_gct = gct; - gct = user; + SET_GCT(user); evacuate(root); - gct = saved_gct; + SET_GCT(saved_gct); } /* ----------------------------------------------------------------------------- @@ -1604,6 +1617,8 @@ resize_generations (void) static void resize_nursery (void) { + lnat min_nursery = RtsFlags.GcFlags.minAllocAreaSize * n_capabilities; + if (RtsFlags.GcFlags.generations == 1) { // Two-space collector: nat blocks; @@ -1646,9 +1661,9 @@ resize_nursery (void) else { blocks *= RtsFlags.GcFlags.oldGenFactor; - if (blocks < RtsFlags.GcFlags.minAllocAreaSize) + if (blocks < min_nursery) { - blocks = RtsFlags.GcFlags.minAllocAreaSize; + blocks = min_nursery; } } resizeNurseries(blocks); @@ -1695,8 +1710,8 @@ resize_nursery (void) (((long)RtsFlags.GcFlags.heapSizeSuggestion - (long)needed) * 100) / (100 + (long)g0s0_pcnt_kept); - if (blocks < (long)RtsFlags.GcFlags.minAllocAreaSize) { - blocks = RtsFlags.GcFlags.minAllocAreaSize; + if (blocks < (long)min_nursery) { + blocks = min_nursery; } resizeNurseries((nat)blocks);