/* -----------------------------------------------------------------------------
*
- * (c) The GHC Team 1998-2006
+ * (c) The GHC Team 1998-2008
*
* Generational garbage collector: evacuation functions
*
#include "MBlock.h"
#include "Evac.h"
#include "GC.h"
+#include "GCThread.h"
#include "GCUtils.h"
#include "Compact.h"
#include "Prelude.h"
#include "LdvProfile.h"
+#if defined(PROF_SPIN) && defined(THREADED_RTS)
+StgWord64 whitehole_spin = 0;
+#endif
+
/* Used to avoid long recursion due to selector thunks
*/
#define MAX_THUNK_SELECTOR_DEPTH 16
{
StgPtr to;
step_workspace *ws;
- bdescr *bd;
/* Find out where we're going, using the handy "to" pointer in
* the step of the source object. If it turns out we need to
}
}
- ws = &gct->steps[stp->gen_no][stp->no];
+ ws = &gct->steps[stp->abs_no];
+ // this compiles to a single mem access to stp->abs_no only
/* chain a new block onto the to-space for the destination step if
* necessary.
*/
- bd = ws->todo_bd;
- to = bd->free;
- if (to + size >= bd->start + BLOCK_SIZE_W) {
- bd = gc_alloc_todo_block(ws);
- to = bd->free;
+ to = ws->todo_free;
+ if (to + size > ws->todo_lim) {
+ to = todo_block_full(size, ws);
}
- bd->free = to + size;
+ ws->todo_free = to + size;
+ ASSERT(ws->todo_free >= ws->todo_bd->free && ws->todo_free <= ws->todo_lim);
return to;
}
-
+
/* -----------------------------------------------------------------------------
The evacuate() code
-------------------------------------------------------------------------- */
-#define MINOR_GC
+#undef PARALLEL_GC
#include "Evac.c-inc"
-#undef MINOR_GC
+#ifdef THREADED_RTS
+#define PARALLEL_GC
#include "Evac.c-inc"
+#endif
/* -----------------------------------------------------------------------------
Evacuate a large object
evacuate_large(StgPtr p)
{
bdescr *bd = Bdescr(p);
- step *stp;
+ step *stp, *new_stp;
step_workspace *ws;
+
+ stp = bd->step;
+ ACQUIRE_SPIN_LOCK(&stp->sync_large_objects);
// object must be at the beginning of the block (or be a ByteArray)
ASSERT(get_itbl((StgClosure *)p)->type == ARR_WORDS ||
/* Don't forget to set the gct->failed_to_evac flag if we didn't get
* the desired destination (see comments in evacuate()).
*/
- if (bd->step < gct->evac_step) {
- gct->failed_to_evac = rtsTrue;
- TICK_GC_FAILED_PROMOTION();
+ if (stp < gct->evac_step) {
+ gct->failed_to_evac = rtsTrue;
+ TICK_GC_FAILED_PROMOTION();
}
+ RELEASE_SPIN_LOCK(&stp->sync_large_objects);
return;
}
- stp = bd->step;
-
- ACQUIRE_SPIN_LOCK(&stp->sync_large_objects);
// remove from large_object list
if (bd->u.back) {
bd->u.back->link = bd->link;
if (bd->link) {
bd->link->u.back = bd->u.back;
}
- RELEASE_SPIN_LOCK(&stp->sync_large_objects);
/* link it on to the evacuated large object list of the destination step
*/
- stp = bd->step->to;
- if (stp < gct->evac_step) {
+ new_stp = stp->to;
+ if (new_stp < gct->evac_step) {
if (gct->eager_promotion) {
- stp = gct->evac_step;
+ new_stp = gct->evac_step;
} else {
gct->failed_to_evac = rtsTrue;
}
}
- ws = &gct->steps[stp->gen_no][stp->no];
- bd->step = stp;
- bd->gen_no = stp->gen_no;
+ ws = &gct->steps[new_stp->abs_no];
+ bd->flags |= BF_EVACUATED;
+ bd->step = new_stp;
+ bd->gen_no = new_stp->gen_no;
bd->link = ws->todo_large_objects;
ws->todo_large_objects = bd;
- bd->flags |= BF_EVACUATED;
+
+ RELEASE_SPIN_LOCK(&stp->sync_large_objects);
}
/* -----------------------------------------------------------------------------
prev = NULL;
while (p)
{
+#ifdef THREADED_RTS
+ ASSERT(p->header.info == &stg_WHITEHOLE_info);
+#else
ASSERT(p->header.info == &stg_BLACKHOLE_info);
+#endif
+ // val must be in to-space. Not always: when we recursively
+ // invoke eval_thunk_selector(), the recursive calls will not
+ // evacuate the value (because we want to select on the value,
+ // not evacuate it), so in this case val is in from-space.
+ // ASSERT(!HEAP_ALLOCED(val) || Bdescr((P_)val)->gen_no > N || (Bdescr((P_)val)->flags & BF_EVACUATED));
+
prev = (StgSelector*)((StgClosure *)p)->payload[0];
// Update the THUNK_SELECTOR with an indirection to the
// EVACUATED closure always points to an object in the
// same or an older generation (required by the short-cut
// test in the EVACUATED case, below).
- SET_INFO(p, &stg_IND_info);
((StgInd *)p)->indirectee = val;
+ write_barrier();
+ SET_INFO(p, &stg_IND_info);
// For the purposes of LDV profiling, we have created an
// indirection.
// save any space in any case, and updating with an indirection is
// trickier in a non-collected gen: we would have to update the
// mutable list.
- if ((bd->gen_no > N) || (bd->flags & BF_EVACUATED)) {
+ if (bd->flags & BF_EVACUATED) {
unchain_thunk_selectors(prev_thunk_selector, (StgClosure *)p);
*q = (StgClosure *)p;
return;
// In threaded mode, we'll use WHITEHOLE to lock the selector
// thunk while we evaluate it.
{
- info_ptr = xchg((StgPtr)&p->header.info, (W_)&stg_WHITEHOLE_info);
- if (info_ptr == (W_)&stg_WHITEHOLE_info) {
- do {
- info_ptr = xchg((StgPtr)&p->header.info, (W_)&stg_WHITEHOLE_info);
- } while (info_ptr == (W_)&stg_WHITEHOLE_info);
- goto bale_out;
- }
- // make sure someone else didn't get here first
- if (INFO_PTR_TO_STRUCT(info_ptr)->type != THUNK_SELECTOR) {
- goto bale_out;
+ do {
+ info_ptr = xchg((StgPtr)&p->header.info, (W_)&stg_WHITEHOLE_info);
+ } while (info_ptr == (W_)&stg_WHITEHOLE_info);
+
+ // make sure someone else didn't get here first...
+ if (IS_FORWARDING_PTR(p) ||
+ INFO_PTR_TO_STRUCT(info_ptr)->type != THUNK_SELECTOR) {
+ // v. tricky now. The THUNK_SELECTOR has been evacuated
+ // by another thread, and is now either a forwarding ptr or IND.
+ // We need to extract ourselves from the current situation
+ // as cleanly as possible.
+ // - unlock the closure
+ // - update *q, we may have done *some* evaluation
+ // - if evac, we need to call evacuate(), because we
+ // need the write-barrier stuff.
+ // - undo the chain we've built to point to p.
+ SET_INFO(p, (const StgInfoTable *)info_ptr);
+ *q = (StgClosure *)p;
+ if (evac) evacuate(q);
+ unchain_thunk_selectors(prev_thunk_selector, (StgClosure *)p);
+ return;
}
}
#else
// from-space during marking, for example. We rely on the property
// that evacuate() doesn't mind if it gets passed a to-space pointer.
- info = get_itbl(selectee);
+ info = (StgInfoTable*)selectee->header.info;
+
+ if (IS_FORWARDING_PTR(info)) {
+ // We don't follow pointers into to-space; the constructor
+ // has already been evacuated, so we won't save any space
+ // leaks by evaluating this selector thunk anyhow.
+ goto bale_out;
+ }
+
+ info = INFO_PTR_TO_STRUCT(info);
switch (info->type) {
case WHITEHOLE:
goto bale_out; // about to be evacuated by another thread (or a loop).
// evaluating until we find the real value, and then
// update the whole chain to point to the value.
val_loop:
- info = get_itbl(UNTAG_CLOSURE(val));
- switch (info->type) {
- case IND:
- case IND_PERM:
- case IND_OLDGEN:
- case IND_OLDGEN_PERM:
- case IND_STATIC:
- val = ((StgInd *)val)->indirectee;
- goto val_loop;
- case THUNK_SELECTOR:
- ((StgClosure*)p)->payload[0] = (StgClosure *)prev_thunk_selector;
- prev_thunk_selector = p;
- p = (StgSelector*)val;
- goto selector_chain;
- default:
- ((StgClosure*)p)->payload[0] = (StgClosure *)prev_thunk_selector;
- prev_thunk_selector = p;
-
- *q = val;
- if (evac) evacuate(q);
- val = *q;
- // evacuate() cannot recurse through
- // eval_thunk_selector(), because we know val is not
- // a THUNK_SELECTOR.
- unchain_thunk_selectors(prev_thunk_selector, val);
- return;
+ info_ptr = (StgWord)UNTAG_CLOSURE(val)->header.info;
+ if (!IS_FORWARDING_PTR(info_ptr))
+ {
+ info = INFO_PTR_TO_STRUCT(info_ptr);
+ switch (info->type) {
+ case IND:
+ case IND_PERM:
+ case IND_OLDGEN:
+ case IND_OLDGEN_PERM:
+ case IND_STATIC:
+ val = ((StgInd *)val)->indirectee;
+ goto val_loop;
+ case THUNK_SELECTOR:
+ ((StgClosure*)p)->payload[0] = (StgClosure *)prev_thunk_selector;
+ prev_thunk_selector = p;
+ p = (StgSelector*)val;
+ goto selector_chain;
+ default:
+ break;
+ }
}
+ ((StgClosure*)p)->payload[0] = (StgClosure *)prev_thunk_selector;
+ prev_thunk_selector = p;
+
+ *q = val;
+ if (evac) evacuate(q);
+ val = *q;
+ // evacuate() cannot recurse through
+ // eval_thunk_selector(), because we know val is not
+ // a THUNK_SELECTOR.
+ unchain_thunk_selectors(prev_thunk_selector, val);
+ return;
}
case IND:
selectee = UNTAG_CLOSURE( ((StgInd *)selectee)->indirectee );
goto selector_loop;
- case EVACUATED:
- // We don't follow pointers into to-space; the constructor
- // has already been evacuated, so we won't save any space
- // leaks by evaluating this selector thunk anyhow.
- goto bale_out;
-
case THUNK_SELECTOR:
{
StgClosure *val;
// We didn't manage to evaluate this thunk; restore the old info
// pointer. But don't forget: we still need to evacuate the thunk itself.
SET_INFO(p, (const StgInfoTable *)info_ptr);
+ // THREADED_RTS: we just unlocked the thunk, so another thread
+ // might get in and update it. copy() will lock it again and
+ // check whether it was updated in the meantime.
+ *q = (StgClosure *)p;
if (evac) {
- copy(&val,(StgClosure *)p,THUNK_SELECTOR_sizeW(),bd->step->to);
- } else {
- val = (StgClosure *)p;
+ copy(q,(const StgInfoTable *)info_ptr,(StgClosure *)p,THUNK_SELECTOR_sizeW(),bd->step->to);
}
- *q = val;
- unchain_thunk_selectors(prev_thunk_selector, val);
+ unchain_thunk_selectors(prev_thunk_selector, *q);
return;
}
projects / ghc-hetmet.git / blobdiff