X-Git-Url: http://git.megacz.com/?p=ghc-hetmet.git;a=blobdiff_plain;f=rts%2Fsm%2FGC.c;h=02fd6d91610b363f3c09018c13b2ef0972b7c73f;hp=e44a310f452c46caf026c7f686b281b69a6f6086;hb=a2a67cd520b9841114d69a87a423dabcb3b4368e;hpb=1aaac3473d8fce705610e4a6437283f7401a5583 diff --git a/rts/sm/GC.c b/rts/sm/GC.c index e44a310..02fd6d9 100644 --- a/rts/sm/GC.c +++ b/rts/sm/GC.c @@ -11,28 +11,23 @@ * * ---------------------------------------------------------------------------*/ -// #include "PosixSource.h" +#include "PosixSource.h" #include "Rts.h" -#include "RtsFlags.h" +#include "HsFFI.h" + +#include "Storage.h" #include "RtsUtils.h" #include "Apply.h" -#include "OSThreads.h" -#include "LdvProfile.h" #include "Updates.h" #include "Stats.h" #include "Schedule.h" #include "Sanity.h" #include "BlockAlloc.h" -#include "MBlock.h" #include "ProfHeap.h" -#include "SchedAPI.h" #include "Weak.h" #include "Prelude.h" -#include "ParTicky.h" // ToDo: move into Rts.h #include "RtsSignals.h" #include "STM.h" -#include "HsFFI.h" -#include "Linker.h" #if defined(RTS_GTK_FRONTPANEL) #include "FrontPanel.h" #endif @@ -40,6 +35,7 @@ #include "RetainerProfile.h" #include "RaiseAsync.h" #include "Papi.h" +#include "Stable.h" #include "GC.h" #include "GCThread.h" @@ -116,7 +112,10 @@ nat mutlist_MUTVARS, /* Thread-local data for each GC thread */ gc_thread **gc_threads = NULL; -// gc_thread *gct = NULL; // this thread's gct TODO: make thread-local + +#if !defined(THREADED_RTS) +StgWord8 the_gc_thread[sizeof(gc_thread) + 64 * sizeof(step_workspace)]; +#endif // Number of threads running in *this* GC. Affects how many // step->todos[] lists we have to look in to find work. @@ -125,6 +124,8 @@ nat n_gc_threads; // For stats: long copied; // *words* copied & scavenged during this GC +rtsBool work_stealing; + DECLARE_GCT /* ----------------------------------------------------------------------------- @@ -142,11 +143,10 @@ static void resize_generations (void); static void resize_nursery (void); static void start_gc_threads (void); static void scavenge_until_all_done (void); -static nat inc_running (void); -static nat dec_running (void); +static StgWord inc_running (void); +static StgWord dec_running (void); static void wakeup_gc_threads (nat n_threads, nat me); static void shutdown_gc_threads (nat n_threads, nat me); -static void continue_gc_threads (nat n_threads, nat me); #if 0 && defined(DEBUG) static void gcCAFs (void); @@ -178,7 +178,7 @@ StgPtr oldgen_scan; void GarbageCollect (rtsBool force_major_gc, nat gc_type USED_IF_THREADS, - Capability *cap USED_IF_THREADS) + Capability *cap) { bdescr *bd; step *stp; @@ -211,6 +211,9 @@ GarbageCollect (rtsBool force_major_gc, // tell the STM to discard any cached closures it's hoping to re-use stmPreGCHook(); + // lock the StablePtr table + stablePtrPreGC(); + #ifdef DEBUG mutlist_MUTVARS = 0; mutlist_MUTARRS = 0; @@ -232,6 +235,19 @@ GarbageCollect (rtsBool force_major_gc, */ n = initialise_N(force_major_gc); +#if defined(THREADED_RTS) + work_stealing = RtsFlags.ParFlags.parGcLoadBalancing; + // It's not always a good idea to do load balancing in parallel + // GC. In particular, for a parallel program we don't want to + // lose locality by moving cached data into another CPU's cache + // (this effect can be quite significant). + // + // We could have a more complex way to deterimine whether to do + // work stealing or not, e.g. it might be a good idea to do it + // if the heap is big. For now, we just turn it on or off with + // a flag. +#endif + /* Start threads, so they can be spinning up while we finish initialisation. */ start_gc_threads(); @@ -250,7 +266,7 @@ GarbageCollect (rtsBool force_major_gc, n_gc_threads = 1; #endif - trace(TRACE_gc|DEBUG_gc, "GC (gen %d): %d KB to collect, %ld MB in use, using %d thread(s)", + debugTrace(DEBUG_gc, "GC (gen %d): %d KB to collect, %ld MB in use, using %d thread(s)", N, n * (BLOCK_SIZE / 1024), mblocks_allocated, n_gc_threads); #ifdef RTS_GTK_FRONTPANEL @@ -300,12 +316,12 @@ GarbageCollect (rtsBool force_major_gc, // this is the main thread #ifdef THREADED_RTS if (n_gc_threads == 1) { - gct = gc_threads[0]; + SET_GCT(gc_threads[0]); } else { - gct = gc_threads[cap->no]; + SET_GCT(gc_threads[cap->no]); } #else - gct = gc_threads[0]; +SET_GCT(gc_threads[0]); #endif /* ----------------------------------------------------------------------- @@ -520,12 +536,12 @@ GarbageCollect (rtsBool force_major_gc, nat i; for (i=0; i < n_gc_threads; i++) { if (n_gc_threads > 1) { - trace(TRACE_gc,"thread %d:", i); - trace(TRACE_gc," copied %ld", gc_threads[i]->copied * sizeof(W_)); - trace(TRACE_gc," scanned %ld", gc_threads[i]->scanned * sizeof(W_)); - trace(TRACE_gc," any_work %ld", gc_threads[i]->any_work); - trace(TRACE_gc," no_work %ld", gc_threads[i]->no_work); - trace(TRACE_gc," scav_find_work %ld", gc_threads[i]->scav_find_work); + debugTrace(DEBUG_gc,"thread %d:", i); + debugTrace(DEBUG_gc," copied %ld", gc_threads[i]->copied * sizeof(W_)); + debugTrace(DEBUG_gc," scanned %ld", gc_threads[i]->scanned * sizeof(W_)); + debugTrace(DEBUG_gc," any_work %ld", gc_threads[i]->any_work); + debugTrace(DEBUG_gc," no_work %ld", gc_threads[i]->no_work); + debugTrace(DEBUG_gc," scav_find_work %ld", gc_threads[i]->scav_find_work); } copied += gc_threads[i]->copied; max_copied = stg_max(gc_threads[i]->copied, max_copied); @@ -734,7 +750,7 @@ GarbageCollect (rtsBool force_major_gc, // start any pending finalizers RELEASE_SM_LOCK; - scheduleFinalizers(last_free_capability, old_weak_ptr_list); + scheduleFinalizers(cap, old_weak_ptr_list); ACQUIRE_SM_LOCK; // send exceptions to any threads which were about to die @@ -750,7 +766,7 @@ GarbageCollect (rtsBool force_major_gc, IF_DEBUG(sanity, checkSanity()); // extra GC trace info - if (traceClass(TRACE_gc|DEBUG_gc)) statDescribeGens(); + IF_DEBUG(gc, statDescribeGens()); #ifdef DEBUG // symbol-table based profiling @@ -777,6 +793,9 @@ GarbageCollect (rtsBool force_major_gc, slop = calcLiveBlocks() * BLOCK_SIZE_W - live; stat_endGC(allocated, live, copied, N, max_copied, avg_copied, slop); + // unlock the StablePtr table + stablePtrPostGC(); + // Guess which generation we'll collect *next* time initialise_N(force_major_gc); @@ -787,11 +806,9 @@ GarbageCollect (rtsBool force_major_gc, } #endif - continue_gc_threads(n_gc_threads, gct->thread_index); - RELEASE_SM_LOCK; - gct = saved_gct; + SET_GCT(saved_gct); } /* ----------------------------------------------------------------------------- @@ -845,15 +862,11 @@ initialise_N (rtsBool force_major_gc) #define GC_THREAD_RUNNING 2 #define GC_THREAD_WAITING_TO_CONTINUE 3 -static gc_thread * -alloc_gc_thread (int n) +static void +new_gc_thread (nat n, gc_thread *t) { nat s; step_workspace *ws; - gc_thread *t; - - t = stgMallocBytes(sizeof(gc_thread) + total_steps * sizeof(step_workspace), - "alloc_gc_thread"); #ifdef THREADED_RTS t->id = 0; @@ -879,10 +892,12 @@ alloc_gc_thread (int n) ws = &t->steps[s]; ws->step = &all_steps[s]; ASSERT(s == ws->step->abs_no); - ws->gct = t; + ws->my_gct = t; ws->todo_bd = NULL; - ws->buffer_todo_bd = NULL; + ws->todo_q = newWSDeque(128); + ws->todo_overflow = NULL; + ws->n_todo_overflow = 0; ws->part_list = NULL; ws->n_part_blocks = 0; @@ -890,8 +905,6 @@ alloc_gc_thread (int n) ws->scavd_list = NULL; ws->n_scavd_blocks = 0; } - - return t; } @@ -906,14 +919,34 @@ initGcThreads (void) "alloc_gc_threads"); for (i = 0; i < RtsFlags.ParFlags.nNodes; i++) { - gc_threads[i] = alloc_gc_thread(i); + gc_threads[i] = + stgMallocBytes(sizeof(gc_thread) + total_steps * sizeof(step_workspace), + "alloc_gc_threads"); + + new_gc_thread(i, gc_threads[i]); } #else - gc_threads = stgMallocBytes (sizeof(gc_thread*), - "alloc_gc_threads"); + gc_threads = stgMallocBytes (sizeof(gc_thread*),"alloc_gc_threads"); + gc_threads[0] = gct; + new_gc_thread(0,gc_threads[0]); +#endif + } +} - gc_threads[0] = alloc_gc_thread(0); +void +freeGcThreads (void) +{ + if (gc_threads != NULL) { +#if defined(THREADED_RTS) + nat i; + for (i = 0; i < RtsFlags.ParFlags.nNodes; i++) { + stgFree (gc_threads[i]); + } + stgFree (gc_threads); +#else + stgFree (gc_threads); #endif + gc_threads = NULL; } } @@ -921,32 +954,22 @@ initGcThreads (void) Start GC threads ------------------------------------------------------------------------- */ -static nat gc_running_threads; +static volatile StgWord gc_running_threads; -#if defined(THREADED_RTS) -static Mutex gc_running_mutex; -#endif - -static nat +static StgWord inc_running (void) { - nat n_running; - ACQUIRE_LOCK(&gc_running_mutex); - n_running = ++gc_running_threads; - RELEASE_LOCK(&gc_running_mutex); - ASSERT(n_running <= n_gc_threads); - return n_running; + StgWord new; + new = atomic_inc(&gc_running_threads); + ASSERT(new <= n_gc_threads); + return new; } -static nat +static StgWord dec_running (void) { - nat n_running; - ACQUIRE_LOCK(&gc_running_mutex); - ASSERT(n_gc_threads != 0); - n_running = --gc_running_threads; - RELEASE_LOCK(&gc_running_mutex); - return n_running; + ASSERT(gc_running_threads != 0); + return atomic_dec(&gc_running_threads); } static rtsBool @@ -974,8 +997,23 @@ any_work (void) } ws = &gct->steps[s]; if (ws->todo_large_objects) return rtsTrue; - if (ws->step->todos) return rtsTrue; + if (!looksEmptyWSDeque(ws->todo_q)) return rtsTrue; + if (ws->todo_overflow) return rtsTrue; + } + +#if defined(THREADED_RTS) + if (work_stealing) { + nat n; + // look for work to steal + for (n = 0; n < n_gc_threads; n++) { + if (n == gct->thread_index) continue; + for (s = total_steps-1; s >= 0; s--) { + ws = &gc_threads[n]->steps[s]; + if (!looksEmptyWSDeque(ws->todo_q)) return rtsTrue; + } + } } +#endif gct->no_work++; @@ -1004,18 +1042,18 @@ loop: r = dec_running(); debugTrace(DEBUG_gc, "GC thread %d idle (%d still running)", - gct->thread_index, r); - + gct->thread_index, r); + while (gc_running_threads != 0) { // usleep(1); - if (any_work()) { - inc_running(); - goto loop; - } - // any_work() does not remove the work from the queue, it - // just checks for the presence of work. If we find any, - // then we increment gc_running_threads and go back to - // scavenge_loop() to perform any pending work. + if (any_work()) { + inc_running(); + goto loop; + } + // any_work() does not remove the work from the queue, it + // just checks for the presence of work. If we find any, + // then we increment gc_running_threads and go back to + // scavenge_loop() to perform any pending work. } // All threads are now stopped @@ -1070,10 +1108,11 @@ gcWorkerThread (Capability *cap) #endif +#if defined(THREADED_RTS) + void waitForGcThreads (Capability *cap USED_IF_THREADS) { -#if defined(THREADED_RTS) nat n_threads = RtsFlags.ParFlags.nNodes; nat me = cap->no; nat i, j; @@ -1099,15 +1138,15 @@ waitForGcThreads (Capability *cap USED_IF_THREADS) if (!retry) break; } } -#endif } +#endif // THREADED_RTS + static void start_gc_threads (void) { #if defined(THREADED_RTS) gc_running_threads = 0; - initMutex(&gc_running_mutex); #endif } @@ -1144,21 +1183,24 @@ shutdown_gc_threads (nat n_threads USED_IF_THREADS, nat me USED_IF_THREADS) #endif } -static void -continue_gc_threads (nat n_threads USED_IF_THREADS, nat me USED_IF_THREADS) -{ #if defined(THREADED_RTS) +void +releaseGCThreads (Capability *cap USED_IF_THREADS) +{ + nat n_threads = RtsFlags.ParFlags.nNodes; + nat me = cap->no; nat i; for (i=0; i < n_threads; i++) { if (i == me) continue; - if (gc_threads[i]->wakeup != GC_THREAD_WAITING_TO_CONTINUE) barf("continue_gc_threads"); + if (gc_threads[i]->wakeup != GC_THREAD_WAITING_TO_CONTINUE) + barf("releaseGCThreads"); gc_threads[i]->wakeup = GC_THREAD_INACTIVE; ACQUIRE_SPIN_LOCK(&gc_threads[i]->gc_spin); RELEASE_SPIN_LOCK(&gc_threads[i]->mut_spin); } -#endif } +#endif /* ---------------------------------------------------------------------------- Initialise a generation that is to be collected @@ -1207,11 +1249,6 @@ init_collected_gen (nat g, nat n_threads) stp->n_words = 0; stp->live_estimate = 0; - // we don't have any to-be-scavenged blocks yet - stp->todos = NULL; - stp->todos_last = NULL; - stp->n_todos = 0; - // initialise the large object queues. stp->scavenged_large_objects = NULL; stp->n_scavenged_large_blocks = 0; @@ -1284,9 +1321,12 @@ init_collected_gen (nat g, nat n_threads) // allocate the first to-space block; extra blocks will be // chained on as necessary. ws->todo_bd = NULL; - ws->buffer_todo_bd = NULL; + ASSERT(looksEmptyWSDeque(ws->todo_q)); alloc_todo_block(ws,0); + ws->todo_overflow = NULL; + ws->n_todo_overflow = 0; + ws->scavd_list = NULL; ws->n_scavd_blocks = 0; } @@ -1329,7 +1369,7 @@ init_uncollected_gen (nat g, nat threads) for (t = 0; t < threads; t++) { ws = &gc_threads[t]->steps[g * RtsFlags.GcFlags.steps + s]; - ws->buffer_todo_bd = NULL; + ASSERT(looksEmptyWSDeque(ws->todo_q)); ws->todo_large_objects = NULL; ws->part_list = NULL; @@ -1405,7 +1445,7 @@ init_gc_thread (gc_thread *t) -------------------------------------------------------------------------- */ static void -mark_root(void *user, StgClosure **root) +mark_root(void *user USED_IF_THREADS, StgClosure **root) { // we stole a register for gct, but this function is called from // *outside* the GC where the register variable is not in effect, @@ -1414,11 +1454,11 @@ mark_root(void *user, StgClosure **root) // incorrect. gc_thread *saved_gct; saved_gct = gct; - gct = user; + SET_GCT(user); evacuate(root); - gct = saved_gct; + SET_GCT(saved_gct); } /* -----------------------------------------------------------------------------