The evacuate() code
-------------------------------------------------------------------------- */
-STATIC_INLINE void
+STATIC_INLINE GNUC_ATTR_HOT void
copy_tag(StgClosure **p, const StgInfoTable *info,
StgClosure *src, nat size, step *stp, StgWord tag)
{
/* Copy wrappers that don't tag the closure after copying */
-STATIC_INLINE void
+STATIC_INLINE GNUC_ATTR_HOT void
copy(StgClosure **p, const StgInfoTable *info,
StgClosure *src, nat size, step *stp)
{
copy_tag(p,info,src,size,stp,0);
}
+/* -----------------------------------------------------------------------------
+ Evacuate a large object
+
+ This just consists of removing the object from the (doubly-linked)
+ step->large_objects list, and linking it on to the (singly-linked)
+ step->new_large_objects list, from where it will be scavenged later.
+
+ Convention: bd->flags has BF_EVACUATED set for a large object
+ that has been evacuated, or unset otherwise.
+ -------------------------------------------------------------------------- */
+
+STATIC_INLINE void
+evacuate_large(StgPtr p)
+{
+ bdescr *bd = Bdescr(p);
+ step *stp, *new_stp;
+ step_workspace *ws;
+
+ stp = bd->step;
+ ACQUIRE_SPIN_LOCK(&stp->sync_large_objects);
+
+ // object must be at the beginning of the block (or be a ByteArray)
+ ASSERT(get_itbl((StgClosure *)p)->type == ARR_WORDS ||
+ (((W_)p & BLOCK_MASK) == 0));
+
+ // already evacuated?
+ if (bd->flags & BF_EVACUATED) {
+ /* Don't forget to set the gct->failed_to_evac flag if we didn't get
+ * the desired destination (see comments in evacuate()).
+ */
+ if (stp < gct->evac_step) {
+ gct->failed_to_evac = rtsTrue;
+ TICK_GC_FAILED_PROMOTION();
+ }
+ RELEASE_SPIN_LOCK(&stp->sync_large_objects);
+ return;
+ }
+
+ // remove from large_object list
+ if (bd->u.back) {
+ bd->u.back->link = bd->link;
+ } else { // first object in the list
+ stp->large_objects = bd->link;
+ }
+ if (bd->link) {
+ bd->link->u.back = bd->u.back;
+ }
+
+ /* link it on to the evacuated large object list of the destination step
+ */
+ new_stp = stp->to;
+ if (new_stp < gct->evac_step) {
+ if (gct->eager_promotion) {
+ new_stp = gct->evac_step;
+ } else {
+ gct->failed_to_evac = rtsTrue;
+ }
+ }
+
+ ws = &gct->steps[new_stp->abs_no];
+ bd->flags |= BF_EVACUATED;
+ bd->step = new_stp;
+ bd->gen_no = new_stp->gen_no;
+ bd->link = ws->todo_large_objects;
+ ws->todo_large_objects = bd;
+
+ RELEASE_SPIN_LOCK(&stp->sync_large_objects);
+}
+
/* ----------------------------------------------------------------------------
Evacuate
extra reads/writes than we save.
------------------------------------------------------------------------- */
-REGPARM1 void
+REGPARM1 GNUC_ATTR_HOT void
evacuate(StgClosure **p)
{
bdescr *bd = NULL;
}
/* -----------------------------------------------------------------------------
- Evacuate a large object
-
- This just consists of removing the object from the (doubly-linked)
- step->large_objects list, and linking it on to the (singly-linked)
- step->new_large_objects list, from where it will be scavenged later.
-
- Convention: bd->flags has BF_EVACUATED set for a large object
- that has been evacuated, or unset otherwise.
- -------------------------------------------------------------------------- */
-
-STATIC_INLINE void
-evacuate_large(StgPtr p)
-{
- bdescr *bd = Bdescr(p);
- step *stp, *new_stp;
- step_workspace *ws;
-
- stp = bd->step;
- ACQUIRE_SPIN_LOCK(&stp->sync_large_objects);
-
- // object must be at the beginning of the block (or be a ByteArray)
- ASSERT(get_itbl((StgClosure *)p)->type == ARR_WORDS ||
- (((W_)p & BLOCK_MASK) == 0));
-
- // already evacuated?
- if (bd->flags & BF_EVACUATED) {
- /* Don't forget to set the gct->failed_to_evac flag if we didn't get
- * the desired destination (see comments in evacuate()).
- */
- if (stp < gct->evac_step) {
- gct->failed_to_evac = rtsTrue;
- TICK_GC_FAILED_PROMOTION();
- }
- RELEASE_SPIN_LOCK(&stp->sync_large_objects);
- return;
- }
-
- // remove from large_object list
- if (bd->u.back) {
- bd->u.back->link = bd->link;
- } else { // first object in the list
- stp->large_objects = bd->link;
- }
- if (bd->link) {
- bd->link->u.back = bd->u.back;
- }
-
- /* link it on to the evacuated large object list of the destination step
- */
- new_stp = stp->to;
- if (new_stp < gct->evac_step) {
- if (gct->eager_promotion) {
- new_stp = gct->evac_step;
- } else {
- gct->failed_to_evac = rtsTrue;
- }
- }
-
- ws = &gct->steps[new_stp->abs_no];
- bd->flags |= BF_EVACUATED;
- bd->step = new_stp;
- bd->gen_no = new_stp->gen_no;
- bd->link = ws->todo_large_objects;
- ws->todo_large_objects = bd;
-
- RELEASE_SPIN_LOCK(&stp->sync_large_objects);
-}
-
-/* -----------------------------------------------------------------------------
Evaluate a THUNK_SELECTOR if possible.
p points to a THUNK_SELECTOR that we want to evaluate. The
prev = (StgSelector*)((StgClosure *)p)->payload[0];
// Update the THUNK_SELECTOR with an indirection to the
- // EVACUATED closure now at p. Why do this rather than
- // upd_evacuee(q,p)? Because we have an invariant that an
- // EVACUATED closure always points to an object in the
- // same or an older generation (required by the short-cut
- // test in the EVACUATED case, below).
- ((StgInd *)p)->indirectee = val;
- write_barrier();
- SET_INFO(p, &stg_IND_info);
+ // value. The value is still in from-space at this stage.
+ //
+ // (old note: Why not do upd_evacuee(q,p)? Because we have an
+ // invariant that an EVACUATED closure always points to an
+ // object in the same or an older generation (required by
+ // the short-cut test in the EVACUATED case, below).
+ if ((StgClosure *)p == val) {
+ // must be a loop; just leave a BLACKHOLE in place. This
+ // can happen when we have a chain of selectors that
+ // eventually loops back on itself. We can't leave an
+ // indirection pointing to itself, and we want the program
+ // to deadlock if it ever enters this closure, so
+ // BLACKHOLE is correct.
+ SET_INFO(p, &stg_BLACKHOLE_info);
+ } else {
+ ((StgInd *)p)->indirectee = val;
+ write_barrier();
+ SET_INFO(p, &stg_IND_info);
+ }
// For the purposes of LDV profiling, we have created an
// indirection.
prev_thunk_selector = p;
*q = val;
- if (evac) evacuate(q);
- val = *q;
+
+ // update the other selectors in the chain *before*
+ // evacuating the value. This is necessary in the case
+ // where the value turns out to be one of the selectors
+ // in the chain (i.e. we have a loop), and evacuating it
+ // would corrupt the chain.
+ unchain_thunk_selectors(prev_thunk_selector, val);
+
// evacuate() cannot recurse through
// eval_thunk_selector(), because we know val is not
// a THUNK_SELECTOR.
- unchain_thunk_selectors(prev_thunk_selector, val);
+ if (evac) evacuate(q);
return;
}