#include "ParTicky.h" // ToDo: move into Rts.h
#include "GCCompact.h"
#include "Signals.h"
+#include "STM.h"
#if defined(GRAN) || defined(PAR)
# include "GranSimRts.h"
# include "ParallelRts.h"
blockUserSignals();
#endif
+ // tell the STM to discard any cached closures its hoping to re-use
+ stmPreGCHook();
+
// tell the stats department that we've started a GC
stat_startGC();
prev = &old_all_threads;
for (t = old_all_threads; t != END_TSO_QUEUE; t = next) {
- (StgClosure *)tmp = isAlive((StgClosure *)t);
+ tmp = (StgTSO *)isAlive((StgClosure *)t);
if (tmp != NULL) {
t = tmp;
StgTSO *t, *tmp, *next;
for (t = old_all_threads; t != END_TSO_QUEUE; t = next) {
next = t->global_link;
- (StgClosure *)tmp = evacuate((StgClosure *)t);
+ tmp = (StgTSO *)evacuate((StgClosure *)t);
tmp->global_link = resurrected_threads;
resurrected_threads = tmp;
}
// w might be WEAK, EVACUATED, or DEAD_WEAK (actually CON_STATIC) here
ASSERT(w->header.info == &stg_DEAD_WEAK_info
|| get_itbl(w)->type == WEAK || get_itbl(w)->type == EVACUATED);
- (StgClosure *)w = evacuate((StgClosure *)w);
+ w = (StgWeak *)evacuate((StgClosure *)w);
*last_w = w;
last_w = &(w->link);
}
case UPDATE_FRAME:
case STOP_FRAME:
case CATCH_FRAME:
+ case CATCH_STM_FRAME:
+ case CATCH_RETRY_FRAME:
+ case ATOMICALLY_FRAME:
// shouldn't see these
barf("evacuate: stack frame at %p\n", q);
return to;
#endif
+ case TREC_HEADER:
+ return copy(q,sizeofW(StgTRecHeader),stp);
+
+ case TVAR_WAIT_QUEUE:
+ return copy(q,sizeofW(StgTVarWaitQueue),stp);
+
+ case TVAR:
+ return copy(q,sizeofW(StgTVar),stp);
+
+ case TREC_CHUNK:
+ return copy(q,sizeofW(StgTRecChunk),stp);
+
default:
barf("evacuate: strange closure type %d", (int)(info->type));
}
scavengeTSO (StgTSO *tso)
{
// chase the link field for any TSOs on the same queue
- (StgClosure *)tso->link = evacuate((StgClosure *)tso->link);
+ tso->link = (StgTSO *)evacuate((StgClosure *)tso->link);
if ( tso->why_blocked == BlockedOnMVar
|| tso->why_blocked == BlockedOnBlackHole
|| tso->why_blocked == BlockedOnException
(StgTSO *)evacuate((StgClosure *)tso->blocked_exceptions);
}
+ // scavange current transaction record
+ tso->trec = (StgTRecHeader *)evacuate((StgClosure *)tso->trec);
+
// scavenge this thread's stack
scavenge_stack(tso->sp, &(tso->stack[tso->stack_size]));
}
small_bitmap:
while (size > 0) {
if ((bitmap & 1) == 0) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
p++;
bitmap = bitmap >> 1;
size = pap->n_args;
while (size > 0) {
if ((bitmap & 1) == 0) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
p++;
bitmap = bitmap >> 1;
{
StgMVar *mvar = ((StgMVar *)p);
evac_gen = 0;
- (StgClosure *)mvar->head = evacuate((StgClosure *)mvar->head);
- (StgClosure *)mvar->tail = evacuate((StgClosure *)mvar->tail);
- (StgClosure *)mvar->value = evacuate((StgClosure *)mvar->value);
+ mvar->head = (StgTSO *)evacuate((StgClosure *)mvar->head);
+ mvar->tail = (StgTSO *)evacuate((StgClosure *)mvar->tail);
+ mvar->value = evacuate((StgClosure *)mvar->value);
evac_gen = saved_evac_gen;
recordMutable((StgMutClosure *)mvar);
failed_to_evac = rtsFalse; // mutable.
end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
p += info->layout.payload.nptrs;
break;
case BCO: {
StgBCO *bco = (StgBCO *)p;
- (StgClosure *)bco->instrs = evacuate((StgClosure *)bco->instrs);
- (StgClosure *)bco->literals = evacuate((StgClosure *)bco->literals);
- (StgClosure *)bco->ptrs = evacuate((StgClosure *)bco->ptrs);
- (StgClosure *)bco->itbls = evacuate((StgClosure *)bco->itbls);
+ bco->instrs = (StgArrWords *)evacuate((StgClosure *)bco->instrs);
+ bco->literals = (StgArrWords *)evacuate((StgClosure *)bco->literals);
+ bco->ptrs = (StgMutArrPtrs *)evacuate((StgClosure *)bco->ptrs);
+ bco->itbls = (StgArrWords *)evacuate((StgClosure *)bco->itbls);
p += bco_sizeW(bco);
break;
}
case BLACKHOLE_BQ:
{
StgBlockingQueue *bh = (StgBlockingQueue *)p;
- (StgClosure *)bh->blocking_queue =
- evacuate((StgClosure *)bh->blocking_queue);
+ bh->blocking_queue =
+ (StgTSO *)evacuate((StgClosure *)bh->blocking_queue);
recordMutable((StgMutClosure *)bh);
failed_to_evac = rtsFalse;
p += BLACKHOLE_sizeW();
evac_gen = 0; // repeatedly mutable
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
evac_gen = saved_evac_gen;
recordMutable((StgMutClosure *)q);
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
// it's tempting to recordMutable() if failed_to_evac is
// false, but that breaks some assumptions (eg. every
}
#endif
+ case TVAR_WAIT_QUEUE:
+ {
+ StgTVarWaitQueue *wq = ((StgTVarWaitQueue *) p);
+ evac_gen = 0;
+ wq->waiting_tso = (StgTSO *)evacuate((StgClosure*)wq->waiting_tso);
+ wq->next_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->next_queue_entry);
+ wq->prev_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->prev_queue_entry);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)wq);
+ failed_to_evac = rtsFalse; // mutable
+ p += sizeofW(StgTVarWaitQueue);
+ break;
+ }
+
+ case TVAR:
+ {
+ StgTVar *tvar = ((StgTVar *) p);
+ evac_gen = 0;
+ tvar->current_value = evacuate((StgClosure*)tvar->current_value);
+ tvar->first_wait_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)tvar->first_wait_queue_entry);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)tvar);
+ failed_to_evac = rtsFalse; // mutable
+ p += sizeofW(StgTVar);
+ break;
+ }
+
+ case TREC_HEADER:
+ {
+ StgTRecHeader *trec = ((StgTRecHeader *) p);
+ evac_gen = 0;
+ trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec);
+ trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)trec);
+ failed_to_evac = rtsFalse; // mutable
+ p += sizeofW(StgTRecHeader);
+ break;
+ }
+
+ case TREC_CHUNK:
+ {
+ StgWord i;
+ StgTRecChunk *tc = ((StgTRecChunk *) p);
+ TRecEntry *e = &(tc -> entries[0]);
+ evac_gen = 0;
+ tc->prev_chunk = (StgTRecChunk *)evacuate((StgClosure*)tc->prev_chunk);
+ for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
+ e->tvar = (StgTVar *)evacuate((StgClosure*)e->tvar);
+ e->expected_value = evacuate((StgClosure*)e->expected_value);
+ e->new_value = evacuate((StgClosure*)e->new_value);
+ }
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)tc);
+ failed_to_evac = rtsFalse; // mutable
+ p += sizeofW(StgTRecChunk);
+ break;
+ }
+
default:
barf("scavenge: unimplemented/strange closure type %d @ %p",
info->type, p);
{
StgMVar *mvar = ((StgMVar *)p);
evac_gen = 0;
- (StgClosure *)mvar->head = evacuate((StgClosure *)mvar->head);
- (StgClosure *)mvar->tail = evacuate((StgClosure *)mvar->tail);
- (StgClosure *)mvar->value = evacuate((StgClosure *)mvar->value);
+ mvar->head = (StgTSO *)evacuate((StgClosure *)mvar->head);
+ mvar->tail = (StgTSO *)evacuate((StgClosure *)mvar->tail);
+ mvar->value = evacuate((StgClosure *)mvar->value);
evac_gen = saved_evac_gen;
failed_to_evac = rtsFalse; // mutable.
break;
end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
break;
}
case BCO: {
StgBCO *bco = (StgBCO *)p;
- (StgClosure *)bco->instrs = evacuate((StgClosure *)bco->instrs);
- (StgClosure *)bco->literals = evacuate((StgClosure *)bco->literals);
- (StgClosure *)bco->ptrs = evacuate((StgClosure *)bco->ptrs);
- (StgClosure *)bco->itbls = evacuate((StgClosure *)bco->itbls);
+ bco->instrs = (StgArrWords *)evacuate((StgClosure *)bco->instrs);
+ bco->literals = (StgArrWords *)evacuate((StgClosure *)bco->literals);
+ bco->ptrs = (StgMutArrPtrs *)evacuate((StgClosure *)bco->ptrs);
+ bco->itbls = (StgArrWords *)evacuate((StgClosure *)bco->itbls);
break;
}
case BLACKHOLE_BQ:
{
StgBlockingQueue *bh = (StgBlockingQueue *)p;
- (StgClosure *)bh->blocking_queue =
- evacuate((StgClosure *)bh->blocking_queue);
+ bh->blocking_queue =
+ (StgTSO *)evacuate((StgClosure *)bh->blocking_queue);
failed_to_evac = rtsFalse;
break;
}
evac_gen = 0; // repeatedly mutable
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
evac_gen = saved_evac_gen;
failed_to_evac = rtsFalse; // mutable anyhow.
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
break;
}
StgInfoTable *rip = get_closure_info(p, &size, &ptrs, &nonptrs, &vhs, str);
#endif
StgRBH *rbh = (StgRBH *)p;
- (StgClosure *)rbh->blocking_queue =
- evacuate((StgClosure *)rbh->blocking_queue);
+ bh->blocking_queue =
+ (StgTSO *)evacuate((StgClosure *)bh->blocking_queue);
recordMutable((StgMutClosure *)rbh);
failed_to_evac = rtsFalse; // mutable anyhow.
IF_DEBUG(gc,
}
#endif // PAR
+ case TVAR_WAIT_QUEUE:
+ {
+ StgTVarWaitQueue *wq = ((StgTVarWaitQueue *) p);
+ evac_gen = 0;
+ wq->waiting_tso = (StgTSO *)evacuate((StgClosure*)wq->waiting_tso);
+ wq->next_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->next_queue_entry);
+ wq->prev_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->prev_queue_entry);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)wq);
+ failed_to_evac = rtsFalse; // mutable
+ break;
+ }
+
+ case TVAR:
+ {
+ StgTVar *tvar = ((StgTVar *) p);
+ evac_gen = 0;
+ tvar->current_value = evacuate((StgClosure*)tvar->current_value);
+ tvar->first_wait_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)tvar->first_wait_queue_entry);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)tvar);
+ failed_to_evac = rtsFalse; // mutable
+ break;
+ }
+
+ case TREC_CHUNK:
+ {
+ StgWord i;
+ StgTRecChunk *tc = ((StgTRecChunk *) p);
+ TRecEntry *e = &(tc -> entries[0]);
+ evac_gen = 0;
+ tc->prev_chunk = (StgTRecChunk *)evacuate((StgClosure*)tc->prev_chunk);
+ for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
+ e->tvar = (StgTVar *)evacuate((StgClosure*)e->tvar);
+ e->expected_value = evacuate((StgClosure*)e->expected_value);
+ e->new_value = evacuate((StgClosure*)e->new_value);
+ }
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)tc);
+ failed_to_evac = rtsFalse; // mutable
+ break;
+ }
+
+ case TREC_HEADER:
+ {
+ StgTRecHeader *trec = ((StgTRecHeader *) p);
+ evac_gen = 0;
+ trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec);
+ trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk);
+ evac_gen = saved_evac_gen;
+ recordMutable((StgMutClosure *)trec);
+ failed_to_evac = rtsFalse; // mutable
+ break;
+ }
+
default:
barf("scavenge_mark_stack: unimplemented/strange closure type %d @ %p",
info->type, p);
end = (StgPtr)((StgClosure *)p)->payload + info->layout.payload.ptrs;
for (q = (StgPtr)((StgClosure *)p)->payload; q < end; q++) {
- (StgClosure *)*q = evacuate((StgClosure *)*q);
+ *q = (StgWord)(StgPtr)evacuate((StgClosure *)*q);
}
break;
}
recordMutable((StgMutClosure *)p);
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
evac_gen = saved_evac_gen;
failed_to_evac = rtsFalse;
next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
break;
}
end = (P_)p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (q = (P_)((StgMutArrPtrs *)p)->payload; q < end; q++) {
- (StgClosure *)*q = evacuate((StgClosure *)*q);
+ *q = (StgWord)(StgPtr)evacuate((StgClosure *)*q);
}
continue;
}
evac_gen = gen->no;
end = (P_)p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
for (q = (P_)((StgMutArrPtrs *)p)->payload; q < end; q++) {
- (StgClosure *)*q = evacuate((StgClosure *)*q);
+ *q = (StgWord)(StgPtr)evacuate((StgClosure *)*q);
}
evac_gen = 0;
// Set the mut_link field to NULL, so that we will put this
case MVAR:
{
StgMVar *mvar = (StgMVar *)p;
- (StgClosure *)mvar->head = evacuate((StgClosure *)mvar->head);
- (StgClosure *)mvar->tail = evacuate((StgClosure *)mvar->tail);
- (StgClosure *)mvar->value = evacuate((StgClosure *)mvar->value);
+ mvar->head = (StgTSO *)evacuate((StgClosure *)mvar->head);
+ mvar->tail = (StgTSO *)evacuate((StgClosure *)mvar->tail);
+ mvar->value = evacuate((StgClosure *)mvar->value);
p->mut_link = gen->mut_list;
gen->mut_list = p;
continue;
case BLACKHOLE_BQ:
{
StgBlockingQueue *bh = (StgBlockingQueue *)p;
- (StgClosure *)bh->blocking_queue =
- evacuate((StgClosure *)bh->blocking_queue);
+ bh->blocking_queue =
+ (StgTSO *)evacuate((StgClosure *)bh->blocking_queue);
p->mut_link = gen->mut_list;
gen->mut_list = p;
continue;
}
#endif
+ case TVAR_WAIT_QUEUE:
+ {
+ StgTVarWaitQueue *wq = ((StgTVarWaitQueue *) p);
+ wq->waiting_tso = (StgTSO *)evacuate((StgClosure*)wq->waiting_tso);
+ wq->next_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->next_queue_entry);
+ wq->prev_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)wq->prev_queue_entry);
+ p->mut_link = gen->mut_list;
+ gen->mut_list = p;
+ continue;
+ }
+
+ case TVAR:
+ {
+ StgTVar *tvar = ((StgTVar *) p);
+ tvar->current_value = evacuate((StgClosure*)tvar->current_value);
+ tvar->first_wait_queue_entry = (StgTVarWaitQueue *)evacuate((StgClosure*)tvar->first_wait_queue_entry);
+ p->mut_link = gen->mut_list;
+ gen->mut_list = p;
+ continue;
+ }
+
+ case TREC_CHUNK:
+ {
+ StgWord i;
+ StgTRecChunk *tc = ((StgTRecChunk *) p);
+ TRecEntry *e = &(tc -> entries[0]);
+ tc->prev_chunk = (StgTRecChunk *)evacuate((StgClosure*)tc->prev_chunk);
+ for (i = 0; i < tc -> next_entry_idx; i ++, e++ ) {
+ e->tvar = (StgTVar *)evacuate((StgClosure*)e->tvar);
+ e->expected_value = evacuate((StgClosure*)e->expected_value);
+ e->new_value = evacuate((StgClosure*)e->new_value);
+ }
+ p->mut_link = gen->mut_list;
+ gen->mut_list = p;
+ continue;
+ }
+
+ case TREC_HEADER:
+ {
+ StgTRecHeader *trec = ((StgTRecHeader *) p);
+ trec->enclosing_trec = (StgTRecHeader *)evacuate((StgClosure*)trec->enclosing_trec);
+ trec->current_chunk = (StgTRecChunk *)evacuate((StgClosure*)trec->current_chunk);
+ p->mut_link = gen->mut_list;
+ gen->mut_list = p;
+ continue;
+ }
+
default:
// shouldn't have anything else on the mutables list
barf("scavenge_mutable_list: strange object? %d", (int)(info->type));
next = (P_)p->payload + info->layout.payload.ptrs;
// evacuate the pointers
for (q = (P_)p->payload; q < next; q++) {
- (StgClosure *)*q = evacuate((StgClosure *)*q);
+ *q = (StgWord)(StgPtr)evacuate((StgClosure *)*q);
}
break;
}
bitmap = large_bitmap->bitmap[b];
for (i = 0; i < size; ) {
if ((bitmap & 1) == 0) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
i++;
p++;
{
while (size > 0) {
if ((bitmap & 1) == 0) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
}
p++;
bitmap = bitmap >> 1;
continue;
// small bitmap (< 32 entries, or 64 on a 64-bit machine)
+ case CATCH_STM_FRAME:
+ case CATCH_RETRY_FRAME:
+ case ATOMICALLY_FRAME:
case STOP_FRAME:
case CATCH_FRAME:
case RET_SMALL:
nat size;
p++;
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
bco = (StgBCO *)*p;
p++;
size = BCO_BITMAP_SIZE(bco);
// follow the ptr words
for (size = RET_DYN_PTRS(dyn); size > 0; size--) {
- (StgClosure *)*p = evacuate((StgClosure *)*p);
+ *p = (StgWord)(StgPtr)evacuate((StgClosure *)*p);
p++;
}
continue;
// normal stack frames; do nothing except advance the pointer
default:
- (StgPtr)frame += stack_frame_sizeW(frame);
+ frame = (StgClosure *)((StgPtr)frame + stack_frame_sizeW(frame));
}
}
}
next_gap_start = (void *)((unsigned char*)gap + sizeof(StgUpdateFrame));
chunk_size = (unsigned char*)gap_end - (unsigned char*)next_gap_start;
- (unsigned char*)sp -= chunk_size;
+ sp -= chunk_size;
memmove(sp, next_gap_start, chunk_size);
}