X-Git-Url: http://git.megacz.com/?a=blobdiff_plain;f=rts%2Fsm%2FStorage.c;h=6f6d59101ea6a1db2f4c6762a4a8ccd635c35322;hb=6a405b1efd138a4af4ed93ce4ff173a4c5704512;hp=c987adda27282e3689902216eccd306701e69f70;hpb=f86e7206ea94b48b94fb61007a1c5d55b8c60f45;p=ghc-hetmet.git diff --git a/rts/sm/Storage.c b/rts/sm/Storage.c index c987add..6f6d591 100644 --- a/rts/sm/Storage.c +++ b/rts/sm/Storage.c @@ -1,6 +1,6 @@ /* ----------------------------------------------------------------------------- * - * (c) The GHC Team, 1998-2006 + * (c) The GHC Team, 1998-2008 * * Storage manager front end * @@ -35,6 +35,8 @@ #include #include +#include "ffi.h" + /* * All these globals require sm_mutex to access in THREADED_RTS mode. */ @@ -46,6 +48,8 @@ bdescr *pinned_object_block; /* allocate pinned objects into this block */ nat alloc_blocks; /* number of allocate()d blocks since GC */ nat alloc_blocks_lim; /* approximate limit on alloc_blocks */ +static bdescr *exec_block; + generation *generations = NULL; /* all the generations */ generation *g0 = NULL; /* generation 0, for convenience */ generation *oldest_gen = NULL; /* oldest generation, for convenience */ @@ -87,6 +91,7 @@ initStep (step *stp, int g, int s) stp->blocks = NULL; stp->n_blocks = 0; stp->n_words = 0; + stp->live_estimate = 0; stp->old_blocks = NULL; stp->n_old_blocks = 0; stp->gen = &generations[g]; @@ -95,12 +100,15 @@ initStep (step *stp, int g, int s) stp->n_large_blocks = 0; stp->scavenged_large_objects = NULL; stp->n_scavenged_large_blocks = 0; - stp->is_compacted = 0; + stp->mark = 0; + stp->compact = 0; stp->bitmap = NULL; #ifdef THREADED_RTS initSpinLock(&stp->sync_todo); initSpinLock(&stp->sync_large_objects); #endif + stp->threads = END_TSO_QUEUE; + stp->old_threads = END_TSO_QUEUE; } void @@ -120,7 +128,7 @@ initStorage( void ) * doing something reasonable. */ /* We use the NOT_NULL variant or gcc warns that the test is always true */ - ASSERT(LOOKS_LIKE_INFO_PTR_NOT_NULL(&stg_BLACKHOLE_info)); + ASSERT(LOOKS_LIKE_INFO_PTR_NOT_NULL((StgWord)&stg_BLACKHOLE_info)); ASSERT(LOOKS_LIKE_CLOSURE_PTR(&stg_dummy_ret_closure)); ASSERT(!HEAP_ALLOCED(&stg_dummy_ret_closure)); @@ -228,11 +236,13 @@ initStorage( void ) } /* The oldest generation has one step. */ - if (RtsFlags.GcFlags.compact) { + if (RtsFlags.GcFlags.compact || RtsFlags.GcFlags.sweep) { if (RtsFlags.GcFlags.generations == 1) { - errorBelch("WARNING: compaction is incompatible with -G1; disabled"); + errorBelch("WARNING: compact/sweep is incompatible with -G1; disabled"); } else { - oldest_gen->steps[0].is_compacted = 1; + oldest_gen->steps[0].mark = 1; + if (RtsFlags.GcFlags.compact) + oldest_gen->steps[0].compact = 1; } } @@ -255,15 +265,20 @@ initStorage( void ) alloc_blocks = 0; alloc_blocks_lim = RtsFlags.GcFlags.minAllocAreaSize; + exec_block = NULL; + /* Tell GNU multi-precision pkg about our custom alloc functions */ mp_set_memory_functions(stgAllocForGMP, stgReallocForGMP, stgDeallocForGMP); #ifdef THREADED_RTS initSpinLock(&gc_alloc_block_sync); - initSpinLock(&recordMutableGen_sync); whitehole_spin = 0; #endif + N = 0; + + initGcThreads(); + IF_DEBUG(gc, statDescribeGens()); RELEASE_SM_LOCK; @@ -360,7 +375,7 @@ newCAF(StgClosure* caf) * any more and can use it as a STATIC_LINK. */ ((StgIndStatic *)caf)->saved_info = NULL; - recordMutableGen(caf, oldest_gen); + recordMutableGen(caf, oldest_gen->no); } RELEASE_SM_LOCK; @@ -551,6 +566,22 @@ resizeNurseries (nat blocks) resizeNurseriesFixed(blocks / n_nurseries); } + +/* ----------------------------------------------------------------------------- + move_TSO is called to update the TSO structure after it has been + moved from one place to another. + -------------------------------------------------------------------------- */ + +void +move_TSO (StgTSO *src, StgTSO *dest) +{ + ptrdiff_t diff; + + // relocate the stack pointer... + diff = (StgPtr)dest - (StgPtr)src; // In *words* + dest->sp = (StgPtr)dest->sp + diff; +} + /* ----------------------------------------------------------------------------- The allocate() interface @@ -563,7 +594,7 @@ resizeNurseries (nat blocks) -------------------------------------------------------------------------- */ StgPtr -allocateInGen (generation *g, nat n) +allocateInGen (generation *g, lnat n) { step *stp; bdescr *bd; @@ -578,18 +609,26 @@ allocateInGen (generation *g, nat n) if (n >= LARGE_OBJECT_THRESHOLD/sizeof(W_)) { - nat req_blocks = (lnat)BLOCK_ROUND_UP(n*sizeof(W_)) / BLOCK_SIZE; + lnat req_blocks = (lnat)BLOCK_ROUND_UP(n*sizeof(W_)) / BLOCK_SIZE; // Attempting to allocate an object larger than maxHeapSize // should definitely be disallowed. (bug #1791) if (RtsFlags.GcFlags.maxHeapSize > 0 && req_blocks >= RtsFlags.GcFlags.maxHeapSize) { heapOverflow(); + // heapOverflow() doesn't exit (see #2592), but we aren't + // in a position to do a clean shutdown here: we + // either have to allocate the memory or exit now. + // Allocating the memory would be bad, because the user + // has requested that we not exceed maxHeapSize, so we + // just exit. + stg_exit(EXIT_HEAPOVERFLOW); } bd = allocGroup(req_blocks); dbl_link_onto(bd, &stp->large_objects); stp->n_large_blocks += bd->blocks; // might be larger than req_blocks + alloc_blocks += bd->blocks; bd->gen_no = g->no; bd->step = stp; bd->flags = BF_LARGE; @@ -620,7 +659,7 @@ allocateInGen (generation *g, nat n) } StgPtr -allocate (nat n) +allocate (lnat n) { return allocateInGen(g0,n); } @@ -639,8 +678,8 @@ allocatedBytes( void ) return allocated; } -// split N blocks off the start of the given bdescr, returning the -// remainder as a new block group. We treat the remainder as if it +// split N blocks off the front of the given bdescr, returning the +// new block group. We treat the remainder as if it // had been freshly allocated in generation 0. bdescr * splitLargeBlock (bdescr *bd, nat blocks) @@ -658,6 +697,7 @@ splitLargeBlock (bdescr *bd, nat blocks) new_bd->step = g0s0; new_bd->flags = BF_LARGE; new_bd->free = bd->free; + ASSERT(new_bd->free <= new_bd->start + new_bd->blocks * BLOCK_SIZE_W); // add the new number of blocks to the counter. Due to the gaps // for block descriptor, new_bd->blocks + bd->blocks might not be @@ -665,7 +705,7 @@ splitLargeBlock (bdescr *bd, nat blocks) bd->step->n_large_blocks += bd->blocks; return new_bd; -} +} /* ----------------------------------------------------------------------------- allocateLocal() @@ -681,7 +721,7 @@ splitLargeBlock (bdescr *bd, nat blocks) -------------------------------------------------------------------------- */ StgPtr -allocateLocal (Capability *cap, nat n) +allocateLocal (Capability *cap, lnat n) { bdescr *bd; StgPtr p; @@ -715,7 +755,9 @@ allocateLocal (Capability *cap, nat n) bd->flags = 0; // NO: alloc_blocks++; // calcAllocated() uses the size of the nursery, and we've - // already bumpted nursery->n_blocks above. + // already bumpted nursery->n_blocks above. We'll GC + // pretty quickly now anyway, because MAYBE_GC() will + // notice that CurrentNursery->link is NULL. } else { // we have a block in the nursery: take it and put // it at the *front* of the nursery list, and use it @@ -758,7 +800,7 @@ allocateLocal (Capability *cap, nat n) ------------------------------------------------------------------------- */ StgPtr -allocatePinned( nat n ) +allocatePinned( lnat n ) { StgPtr p; bdescr *bd = pinned_object_block; @@ -822,6 +864,35 @@ dirty_MUT_VAR(StgRegTable *reg, StgClosure *p) } } +// Setting a TSO's link field with a write barrier. +// It is *not* necessary to call this function when +// * setting the link field to END_TSO_QUEUE +// * putting a TSO on the blackhole_queue +// * setting the link field of the currently running TSO, as it +// will already be dirty. +void +setTSOLink (Capability *cap, StgTSO *tso, StgTSO *target) +{ + bdescr *bd; + if ((tso->flags & (TSO_DIRTY|TSO_LINK_DIRTY)) == 0) { + tso->flags |= TSO_LINK_DIRTY; + bd = Bdescr((StgPtr)tso); + if (bd->gen_no > 0) recordMutableCap((StgClosure*)tso,cap,bd->gen_no); + } + tso->_link = target; +} + +void +dirty_TSO (Capability *cap, StgTSO *tso) +{ + bdescr *bd; + if ((tso->flags & (TSO_DIRTY|TSO_LINK_DIRTY)) == 0) { + bd = Bdescr((StgPtr)tso); + if (bd->gen_no > 0) recordMutableCap((StgClosure*)tso,cap,bd->gen_no); + } + tso->flags |= TSO_DIRTY; +} + /* This is the write barrier for MVARs. An MVAR_CLEAN objects is not on the mutable list; a MVAR_DIRTY is. When written to, a @@ -875,12 +946,14 @@ stgAllocForGMP (size_t size_in_bytes) static void * stgReallocForGMP (void *ptr, size_t old_size, size_t new_size) { + size_t min_size; void *new_stuff_ptr = stgAllocForGMP(new_size); nat i = 0; char *p = (char *) ptr; char *q = (char *) new_stuff_ptr; - for (; i < old_size; i++, p++, q++) { + min_size = old_size < new_size ? old_size : new_size; + for (; i < min_size; i++, p++, q++) { *q = *p; } @@ -985,6 +1058,7 @@ countOccupied(bdescr *bd) words = 0; for (; bd != NULL; bd = bd->link) { + ASSERT(bd->free <= bd->start + bd->blocks * BLOCK_SIZE_W); words += bd->free - bd->start; } return words; @@ -1032,14 +1106,27 @@ calcNeeded(void) for (s = 0; s < generations[g].n_steps; s++) { if (g == 0 && s == 0) { continue; } stp = &generations[g].steps[s]; + + // we need at least this much space + needed += stp->n_blocks + stp->n_large_blocks; + + // any additional space needed to collect this gen next time? if (g == 0 || // always collect gen 0 (generations[g].steps[0].n_blocks + generations[g].steps[0].n_large_blocks - > generations[g].max_blocks - && stp->is_compacted == 0)) { - needed += 2 * stp->n_blocks + stp->n_large_blocks; - } else { - needed += stp->n_blocks + stp->n_large_blocks; + > generations[g].max_blocks)) { + // we will collect this gen next time + if (stp->mark) { + // bitmap: + needed += stp->n_blocks / BITS_IN(W_); + // mark stack: + needed += stp->n_blocks / 100; + } + if (stp->compact) { + continue; // no additional space needed for compaction + } else { + needed += stp->n_blocks; + } } } } @@ -1066,9 +1153,37 @@ calcNeeded(void) should be modified to use allocateExec instead of VirtualAlloc. ------------------------------------------------------------------------- */ -static bdescr *exec_block; +#if defined(linux_HOST_OS) + +// On Linux we need to use libffi for allocating executable memory, +// because it knows how to work around the restrictions put in place +// by SELinux. -void *allocateExec (nat bytes) +void *allocateExec (nat bytes, void **exec_ret) +{ + void **ret, **exec; + ACQUIRE_SM_LOCK; + ret = ffi_closure_alloc (sizeof(void *) + (size_t)bytes, (void**)&exec); + RELEASE_SM_LOCK; + if (ret == NULL) return ret; + *ret = ret; // save the address of the writable mapping, for freeExec(). + *exec_ret = exec + 1; + return (ret + 1); +} + +// freeExec gets passed the executable address, not the writable address. +void freeExec (void *addr) +{ + void *writable; + writable = *((void**)addr - 1); + ACQUIRE_SM_LOCK; + ffi_closure_free (writable); + RELEASE_SM_LOCK +} + +#else + +void *allocateExec (nat bytes, void **exec_ret) { void *ret; nat n; @@ -1104,6 +1219,7 @@ void *allocateExec (nat bytes) exec_block->free += n + 1; RELEASE_SM_LOCK + *exec_ret = ret; return ret; } @@ -1141,6 +1257,8 @@ void freeExec (void *addr) RELEASE_SM_LOCK } +#endif /* mingw32_HOST_OS */ + /* ----------------------------------------------------------------------------- Debugging @@ -1205,6 +1323,51 @@ stepBlocks (step *stp) countAllocdBlocks(stp->large_objects); } +// If memInventory() calculates that we have a memory leak, this +// function will try to find the block(s) that are leaking by marking +// all the ones that we know about, and search through memory to find +// blocks that are not marked. In the debugger this can help to give +// us a clue about what kind of block leaked. In the future we might +// annotate blocks with their allocation site to give more helpful +// info. +static void +findMemoryLeak (void) +{ + nat g, s, i; + for (g = 0; g < RtsFlags.GcFlags.generations; g++) { + for (i = 0; i < n_capabilities; i++) { + markBlocks(capabilities[i].mut_lists[g]); + } + markBlocks(generations[g].mut_list); + for (s = 0; s < generations[g].n_steps; s++) { + markBlocks(generations[g].steps[s].blocks); + markBlocks(generations[g].steps[s].large_objects); + } + } + + for (i = 0; i < n_nurseries; i++) { + markBlocks(nurseries[i].blocks); + markBlocks(nurseries[i].large_objects); + } + +#ifdef PROFILING + // TODO: + // if (RtsFlags.ProfFlags.doHeapProfile == HEAP_BY_RETAINER) { + // markRetainerBlocks(); + // } +#endif + + // count the blocks allocated by the arena allocator + // TODO: + // markArenaBlocks(); + + // count the blocks containing executable memory + markBlocks(exec_block); + + reportUnmarkedBlocks(); +} + + void memInventory (rtsBool show) { @@ -1261,6 +1424,7 @@ memInventory (rtsBool show) #define MB(n) (((n) * BLOCK_SIZE_W) / ((1024*1024)/sizeof(W_))) leak = live_blocks + free_blocks != mblocks_allocated * BLOCKS_PER_MBLOCK; + if (show || leak) { if (leak) { @@ -1289,6 +1453,13 @@ memInventory (rtsBool show) mblocks_allocated * BLOCKS_PER_MBLOCK, mblocks_allocated); } } + + if (leak) { + debugBelch("\n"); + findMemoryLeak(); + } + ASSERT(n_alloc_blocks == live_blocks); + ASSERT(!leak); } @@ -1327,6 +1498,14 @@ checkSanity( void ) checkFreeListSanity(); } + +#if defined(THREADED_RTS) + // check the stacks too in threaded mode, because we don't do a + // full heap sanity check in this case (see checkHeap()) + checkGlobalTSOList(rtsTrue); +#else + checkGlobalTSOList(rtsFalse); +#endif } /* Nursery sanity check */