1 /* -----------------------------------------------------------------------------
2 * $Id: GC.c,v 1.35 1999/02/17 15:04:40 simonm Exp $
4 * (c) The GHC Team 1998-1999
6 * Generational garbage collector
8 * ---------------------------------------------------------------------------*/
14 #include "StoragePriv.h"
17 #include "SchedAPI.h" /* for ReverCAFs prototype */
20 #include "BlockAlloc.h"
22 #include "DebugProf.h"
25 #include "StablePriv.h"
29 /* STATIC OBJECT LIST.
32 * We maintain a linked list of static objects that are still live.
33 * The requirements for this list are:
35 * - we need to scan the list while adding to it, in order to
36 * scavenge all the static objects (in the same way that
37 * breadth-first scavenging works for dynamic objects).
39 * - we need to be able to tell whether an object is already on
40 * the list, to break loops.
42 * Each static object has a "static link field", which we use for
43 * linking objects on to the list. We use a stack-type list, consing
44 * objects on the front as they are added (this means that the
45 * scavenge phase is depth-first, not breadth-first, but that
48 * A separate list is kept for objects that have been scavenged
49 * already - this is so that we can zero all the marks afterwards.
51 * An object is on the list if its static link field is non-zero; this
52 * means that we have to mark the end of the list with '1', not NULL.
54 * Extra notes for generational GC:
56 * Each generation has a static object list associated with it. When
57 * collecting generations up to N, we treat the static object lists
58 * from generations > N as roots.
60 * We build up a static object list while collecting generations 0..N,
61 * which is then appended to the static object list of generation N+1.
63 StgClosure* static_objects; /* live static objects */
64 StgClosure* scavenged_static_objects; /* static objects scavenged so far */
66 /* N is the oldest generation being collected, where the generations
67 * are numbered starting at 0. A major GC (indicated by the major_gc
68 * flag) is when we're collecting all generations. We only attempt to
69 * deal with static objects and GC CAFs when doing a major GC.
72 static rtsBool major_gc;
74 /* Youngest generation that objects should be evacuated to in
75 * evacuate(). (Logically an argument to evacuate, but it's static
76 * a lot of the time so we optimise it into a global variable).
82 static StgWeak *old_weak_ptr_list; /* also pending finaliser list */
83 static rtsBool weak_done; /* all done for this pass */
85 /* Flag indicating failure to evacuate an object to the desired
88 static rtsBool failed_to_evac;
90 /* Old to-space (used for two-space collector only)
94 /* Data used for allocation area sizing.
96 lnat new_blocks; /* blocks allocated during this GC */
97 lnat g0s0_pcnt_kept = 30; /* percentage of g0s0 live at last minor GC */
99 /* -----------------------------------------------------------------------------
100 Static function declarations
101 -------------------------------------------------------------------------- */
103 static StgClosure *evacuate(StgClosure *q);
104 static void zeroStaticObjectList(StgClosure* first_static);
105 static rtsBool traverse_weak_ptr_list(void);
106 static void zeroMutableList(StgMutClosure *first);
107 static void revertDeadCAFs(void);
109 static void scavenge_stack(StgPtr p, StgPtr stack_end);
110 static void scavenge_large(step *step);
111 static void scavenge(step *step);
112 static void scavenge_static(void);
113 static void scavenge_mutable_list(generation *g);
114 static void scavenge_mut_once_list(generation *g);
117 static void gcCAFs(void);
120 /* -----------------------------------------------------------------------------
123 For garbage collecting generation N (and all younger generations):
125 - follow all pointers in the root set. the root set includes all
126 mutable objects in all steps in all generations.
128 - for each pointer, evacuate the object it points to into either
129 + to-space in the next higher step in that generation, if one exists,
130 + if the object's generation == N, then evacuate it to the next
131 generation if one exists, or else to-space in the current
133 + if the object's generation < N, then evacuate it to to-space
134 in the next generation.
136 - repeatedly scavenge to-space from each step in each generation
137 being collected until no more objects can be evacuated.
139 - free from-space in each step, and set from-space = to-space.
141 -------------------------------------------------------------------------- */
143 void GarbageCollect(void (*get_roots)(void))
147 lnat live, allocated, collected = 0;
151 CostCentreStack *prev_CCS;
154 /* tell the stats department that we've started a GC */
157 /* attribute any costs to CCS_GC */
163 /* We might have been called from Haskell land by _ccall_GC, in
164 * which case we need to call threadPaused() because the scheduler
165 * won't have done it.
167 if (CurrentTSO) { threadPaused(CurrentTSO); }
169 /* Approximate how much we allocated: number of blocks in the
170 * nursery + blocks allocated via allocate() - unused nusery blocks.
171 * This leaves a little slop at the end of each block, and doesn't
172 * take into account large objects (ToDo).
174 allocated = (nursery_blocks * BLOCK_SIZE_W) + allocated_bytes();
175 for ( bd = current_nursery->link; bd != NULL; bd = bd->link ) {
176 allocated -= BLOCK_SIZE_W;
179 /* Figure out which generation to collect
182 for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
183 if (generations[g].steps[0].n_blocks >= generations[g].max_blocks) {
187 major_gc = (N == RtsFlags.GcFlags.generations-1);
189 /* check stack sanity *before* GC (ToDo: check all threads) */
190 /*IF_DEBUG(sanity, checkTSO(MainTSO,0)); */
191 IF_DEBUG(sanity, checkFreeListSanity());
193 /* Initialise the static object lists
195 static_objects = END_OF_STATIC_LIST;
196 scavenged_static_objects = END_OF_STATIC_LIST;
198 /* zero the mutable list for the oldest generation (see comment by
199 * zeroMutableList below).
202 zeroMutableList(generations[RtsFlags.GcFlags.generations-1].mut_once_list);
205 /* Save the old to-space if we're doing a two-space collection
207 if (RtsFlags.GcFlags.generations == 1) {
208 old_to_space = g0s0->to_space;
209 g0s0->to_space = NULL;
212 /* Keep a count of how many new blocks we allocated during this GC
213 * (used for resizing the allocation area, later).
217 /* Initialise to-space in all the generations/steps that we're
220 for (g = 0; g <= N; g++) {
221 generations[g].mut_once_list = END_MUT_LIST;
222 generations[g].mut_list = END_MUT_LIST;
224 for (s = 0; s < generations[g].n_steps; s++) {
226 /* generation 0, step 0 doesn't need to-space */
227 if (g == 0 && s == 0 && RtsFlags.GcFlags.generations > 1) {
231 /* Get a free block for to-space. Extra blocks will be chained on
235 step = &generations[g].steps[s];
236 ASSERT(step->gen->no == g);
237 ASSERT(step->hp ? Bdescr(step->hp)->step == step : rtsTrue);
238 bd->gen = &generations[g];
241 bd->evacuated = 1; /* it's a to-space block */
242 step->hp = bd->start;
243 step->hpLim = step->hp + BLOCK_SIZE_W;
246 step->to_blocks = 1; /* ???? */
247 step->scan = bd->start;
249 step->new_large_objects = NULL;
250 step->scavenged_large_objects = NULL;
251 /* mark the large objects as not evacuated yet */
252 for (bd = step->large_objects; bd; bd = bd->link) {
258 /* make sure the older generations have at least one block to
259 * allocate into (this makes things easier for copy(), see below.
261 for (g = N+1; g < RtsFlags.GcFlags.generations; g++) {
262 for (s = 0; s < generations[g].n_steps; s++) {
263 step = &generations[g].steps[s];
264 if (step->hp_bd == NULL) {
266 bd->gen = &generations[g];
269 bd->evacuated = 0; /* *not* a to-space block */
270 step->hp = bd->start;
271 step->hpLim = step->hp + BLOCK_SIZE_W;
276 /* Set the scan pointer for older generations: remember we
277 * still have to scavenge objects that have been promoted. */
278 step->scan = step->hp;
279 step->scan_bd = step->hp_bd;
280 step->to_space = NULL;
282 step->new_large_objects = NULL;
283 step->scavenged_large_objects = NULL;
287 /* -----------------------------------------------------------------------
288 * follow all the roots that we know about:
289 * - mutable lists from each generation > N
290 * we want to *scavenge* these roots, not evacuate them: they're not
291 * going to move in this GC.
292 * Also: do them in reverse generation order. This is because we
293 * often want to promote objects that are pointed to by older
294 * generations early, so we don't have to repeatedly copy them.
295 * Doing the generations in reverse order ensures that we don't end
296 * up in the situation where we want to evac an object to gen 3 and
297 * it has already been evaced to gen 2.
301 for (g = RtsFlags.GcFlags.generations-1; g > N; g--) {
302 generations[g].saved_mut_list = generations[g].mut_list;
303 generations[g].mut_list = END_MUT_LIST;
306 /* Do the mut-once lists first */
307 for (g = RtsFlags.GcFlags.generations-1; g > N; g--) {
308 scavenge_mut_once_list(&generations[g]);
310 for (st = generations[g].n_steps-1; st >= 0; st--) {
311 scavenge(&generations[g].steps[st]);
315 for (g = RtsFlags.GcFlags.generations-1; g > N; g--) {
316 scavenge_mutable_list(&generations[g]);
318 for (st = generations[g].n_steps-1; st >= 0; st--) {
319 scavenge(&generations[g].steps[st]);
324 /* follow all the roots that the application knows about.
329 /* And don't forget to mark the TSO if we got here direct from
332 CurrentTSO = (StgTSO *)MarkRoot((StgClosure *)CurrentTSO);
335 /* Mark the weak pointer list, and prepare to detect dead weak
339 old_weak_ptr_list = weak_ptr_list;
340 weak_ptr_list = NULL;
341 weak_done = rtsFalse;
343 /* Mark the stable pointer table.
345 markStablePtrTable(major_gc);
349 /* ToDo: To fix the caf leak, we need to make the commented out
350 * parts of this code do something sensible - as described in
353 extern void markHugsObjects(void);
355 /* ToDo: This (undefined) function should contain the scavenge
356 * loop immediately below this block of code - but I'm not sure
357 * enough of the details to do this myself.
359 scavengeEverything();
360 /* revert dead CAFs and update enteredCAFs list */
365 /* This will keep the CAFs and the attached BCOs alive
366 * but the values will have been reverted
368 scavengeEverything();
373 /* -------------------------------------------------------------------------
374 * Repeatedly scavenge all the areas we know about until there's no
375 * more scavenging to be done.
382 /* scavenge static objects */
383 if (major_gc && static_objects != END_OF_STATIC_LIST) {
387 /* When scavenging the older generations: Objects may have been
388 * evacuated from generations <= N into older generations, and we
389 * need to scavenge these objects. We're going to try to ensure that
390 * any evacuations that occur move the objects into at least the
391 * same generation as the object being scavenged, otherwise we
392 * have to create new entries on the mutable list for the older
396 /* scavenge each step in generations 0..maxgen */
400 for (gen = RtsFlags.GcFlags.generations-1; gen >= 0; gen--) {
401 for (st = generations[gen].n_steps-1; st >= 0 ; st--) {
402 step = &generations[gen].steps[st];
404 if (step->hp_bd != step->scan_bd || step->scan < step->hp) {
409 if (step->new_large_objects != NULL) {
410 scavenge_large(step);
417 if (flag) { goto loop; }
419 /* must be last... */
420 if (traverse_weak_ptr_list()) { /* returns rtsTrue if evaced something */
425 /* Now see which stable names are still alive
427 gcStablePtrTable(major_gc);
429 /* Set the maximum blocks for the oldest generation, based on twice
430 * the amount of live data now, adjusted to fit the maximum heap
433 * This is an approximation, since in the worst case we'll need
434 * twice the amount of live data plus whatever space the other
437 if (RtsFlags.GcFlags.generations > 1) {
439 oldest_gen->max_blocks =
440 stg_max(oldest_gen->steps[0].to_blocks * RtsFlags.GcFlags.oldGenFactor,
441 RtsFlags.GcFlags.minOldGenSize);
442 if (oldest_gen->max_blocks > RtsFlags.GcFlags.maxHeapSize / 2) {
443 oldest_gen->max_blocks = RtsFlags.GcFlags.maxHeapSize / 2;
444 if (((int)oldest_gen->max_blocks -
445 (int)oldest_gen->steps[0].to_blocks) <
446 (RtsFlags.GcFlags.pcFreeHeap *
447 RtsFlags.GcFlags.maxHeapSize / 200)) {
454 /* run through all the generations/steps and tidy up
456 for (g = 0; g < RtsFlags.GcFlags.generations; g++) {
459 generations[g].collections++; /* for stats */
462 for (s = 0; s < generations[g].n_steps; s++) {
464 step = &generations[g].steps[s];
466 if (!(g == 0 && s == 0 && RtsFlags.GcFlags.generations > 1)) {
467 /* Tidy the end of the to-space chains */
468 step->hp_bd->free = step->hp;
469 step->hp_bd->link = NULL;
472 /* for generations we collected... */
475 collected += step->n_blocks * BLOCK_SIZE_W; /* for stats */
477 /* free old memory and shift to-space into from-space for all
478 * the collected steps (except the allocation area). These
479 * freed blocks will probaby be quickly recycled.
481 if (!(g == 0 && s == 0)) {
482 freeChain(step->blocks);
483 step->blocks = step->to_space;
484 step->n_blocks = step->to_blocks;
485 step->to_space = NULL;
487 for (bd = step->blocks; bd != NULL; bd = bd->link) {
488 bd->evacuated = 0; /* now from-space */
492 /* LARGE OBJECTS. The current live large objects are chained on
493 * scavenged_large, having been moved during garbage
494 * collection from large_objects. Any objects left on
495 * large_objects list are therefore dead, so we free them here.
497 for (bd = step->large_objects; bd != NULL; bd = next) {
502 for (bd = step->scavenged_large_objects; bd != NULL; bd = bd->link) {
505 step->large_objects = step->scavenged_large_objects;
507 /* Set the maximum blocks for this generation, interpolating
508 * between the maximum size of the oldest and youngest
511 * max_blocks = oldgen_max_blocks * G
512 * ----------------------
517 generations[g].max_blocks = (oldest_gen->max_blocks * g)
518 / (RtsFlags.GcFlags.generations-1);
520 generations[g].max_blocks = oldest_gen->max_blocks;
523 /* for older generations... */
526 /* For older generations, we need to append the
527 * scavenged_large_object list (i.e. large objects that have been
528 * promoted during this GC) to the large_object list for that step.
530 for (bd = step->scavenged_large_objects; bd; bd = next) {
533 dbl_link_onto(bd, &step->large_objects);
536 /* add the new blocks we promoted during this GC */
537 step->n_blocks += step->to_blocks;
542 /* Guess the amount of live data for stats. */
545 /* Two-space collector:
546 * Free the old to-space, and estimate the amount of live data.
548 if (RtsFlags.GcFlags.generations == 1) {
551 if (old_to_space != NULL) {
552 freeChain(old_to_space);
554 for (bd = g0s0->to_space; bd != NULL; bd = bd->link) {
555 bd->evacuated = 0; /* now from-space */
558 /* For a two-space collector, we need to resize the nursery. */
560 /* set up a new nursery. Allocate a nursery size based on a
561 * function of the amount of live data (currently a factor of 2,
562 * should be configurable (ToDo)). Use the blocks from the old
563 * nursery if possible, freeing up any left over blocks.
565 * If we get near the maximum heap size, then adjust our nursery
566 * size accordingly. If the nursery is the same size as the live
567 * data (L), then we need 3L bytes. We can reduce the size of the
568 * nursery to bring the required memory down near 2L bytes.
570 * A normal 2-space collector would need 4L bytes to give the same
571 * performance we get from 3L bytes, reducing to the same
572 * performance at 2L bytes.
574 blocks = g0s0->to_blocks;
576 if ( blocks * RtsFlags.GcFlags.oldGenFactor * 2 >
577 RtsFlags.GcFlags.maxHeapSize ) {
578 int adjusted_blocks; /* signed on purpose */
581 adjusted_blocks = (RtsFlags.GcFlags.maxHeapSize - 2 * blocks);
582 IF_DEBUG(gc, fprintf(stderr, "Near maximum heap size of 0x%x blocks, blocks = %d, adjusted to %d\n", RtsFlags.GcFlags.maxHeapSize, blocks, adjusted_blocks));
583 pc_free = adjusted_blocks * 100 / RtsFlags.GcFlags.maxHeapSize;
584 if (pc_free < RtsFlags.GcFlags.pcFreeHeap) /* might even be < 0 */ {
587 blocks = adjusted_blocks;
590 blocks *= RtsFlags.GcFlags.oldGenFactor;
591 if (blocks < RtsFlags.GcFlags.minAllocAreaSize) {
592 blocks = RtsFlags.GcFlags.minAllocAreaSize;
595 resizeNursery(blocks);
598 /* Generational collector:
599 * If the user has given us a suggested heap size, adjust our
600 * allocation area to make best use of the memory available.
603 if (RtsFlags.GcFlags.heapSizeSuggestion) {
605 nat needed = calcNeeded(); /* approx blocks needed at next GC */
607 /* Guess how much will be live in generation 0 step 0 next time.
608 * A good approximation is the obtained by finding the
609 * percentage of g0s0 that was live at the last minor GC.
612 g0s0_pcnt_kept = (new_blocks * 100) / g0s0->n_blocks;
615 /* Estimate a size for the allocation area based on the
616 * information available. We might end up going slightly under
617 * or over the suggested heap size, but we should be pretty
620 * Formula: suggested - needed
621 * ----------------------------
622 * 1 + g0s0_pcnt_kept/100
624 * where 'needed' is the amount of memory needed at the next
625 * collection for collecting all steps except g0s0.
628 (((int)RtsFlags.GcFlags.heapSizeSuggestion - (int)needed) * 100) /
629 (100 + (int)g0s0_pcnt_kept);
631 if (blocks < (int)RtsFlags.GcFlags.minAllocAreaSize) {
632 blocks = RtsFlags.GcFlags.minAllocAreaSize;
635 resizeNursery((nat)blocks);
639 /* revert dead CAFs and update enteredCAFs list */
642 /* mark the garbage collected CAFs as dead */
644 if (major_gc) { gcCAFs(); }
647 /* zero the scavenged static object list */
649 zeroStaticObjectList(scavenged_static_objects);
654 for (bd = g0s0->blocks; bd; bd = bd->link) {
655 bd->free = bd->start;
656 ASSERT(bd->gen == g0);
657 ASSERT(bd->step == g0s0);
659 current_nursery = g0s0->blocks;
661 /* Free the small objects allocated via allocate(), since this will
662 * all have been copied into G0S1 now.
664 if (small_alloc_list != NULL) {
665 freeChain(small_alloc_list);
667 small_alloc_list = NULL;
669 alloc_blocks_lim = RtsFlags.GcFlags.minAllocAreaSize;
671 /* start any pending finalizers */
672 scheduleFinalizers(old_weak_ptr_list);
674 /* check sanity after GC */
675 IF_DEBUG(sanity, checkSanity(N));
677 /* extra GC trace info */
678 IF_DEBUG(gc, stat_describe_gens());
681 /* symbol-table based profiling */
682 /* heapCensus(to_space); */ /* ToDo */
685 /* restore enclosing cost centre */
690 /* check for memory leaks if sanity checking is on */
691 IF_DEBUG(sanity, memInventory());
693 /* ok, GC over: tell the stats department what happened. */
694 stat_endGC(allocated, collected, live, N);
697 /* -----------------------------------------------------------------------------
700 traverse_weak_ptr_list is called possibly many times during garbage
701 collection. It returns a flag indicating whether it did any work
702 (i.e. called evacuate on any live pointers).
704 Invariant: traverse_weak_ptr_list is called when the heap is in an
705 idempotent state. That means that there are no pending
706 evacuate/scavenge operations. This invariant helps the weak
707 pointer code decide which weak pointers are dead - if there are no
708 new live weak pointers, then all the currently unreachable ones are
711 For generational GC: we just don't try to finalize weak pointers in
712 older generations than the one we're collecting. This could
713 probably be optimised by keeping per-generation lists of weak
714 pointers, but for a few weak pointers this scheme will work.
715 -------------------------------------------------------------------------- */
718 traverse_weak_ptr_list(void)
720 StgWeak *w, **last_w, *next_w;
722 rtsBool flag = rtsFalse;
724 if (weak_done) { return rtsFalse; }
726 /* doesn't matter where we evacuate values/finalizers to, since
727 * these pointers are treated as roots (iff the keys are alive).
731 last_w = &old_weak_ptr_list;
732 for (w = old_weak_ptr_list; w; w = next_w) {
734 if ((new = isAlive(w->key))) {
736 /* evacuate the value and finalizer */
737 w->value = evacuate(w->value);
738 w->finalizer = evacuate(w->finalizer);
739 /* remove this weak ptr from the old_weak_ptr list */
741 /* and put it on the new weak ptr list */
743 w->link = weak_ptr_list;
746 IF_DEBUG(weak, fprintf(stderr,"Weak pointer still alive at %p -> %p\n", w, w->key));
756 /* If we didn't make any changes, then we can go round and kill all
757 * the dead weak pointers. The old_weak_ptr list is used as a list
758 * of pending finalizers later on.
760 if (flag == rtsFalse) {
761 for (w = old_weak_ptr_list; w; w = w->link) {
762 w->value = evacuate(w->value);
763 w->finalizer = evacuate(w->finalizer);
771 /* -----------------------------------------------------------------------------
772 isAlive determines whether the given closure is still alive (after
773 a garbage collection) or not. It returns the new address of the
774 closure if it is alive, or NULL otherwise.
775 -------------------------------------------------------------------------- */
778 isAlive(StgClosure *p)
786 /* ToDo: for static closures, check the static link field.
787 * Problem here is that we sometimes don't set the link field, eg.
788 * for static closures with an empty SRT or CONSTR_STATIC_NOCAFs.
791 /* ignore closures in generations that we're not collecting. */
792 if (LOOKS_LIKE_STATIC(p) || Bdescr((P_)p)->gen->no > N) {
796 switch (info->type) {
801 case IND_OLDGEN: /* rely on compatible layout with StgInd */
802 case IND_OLDGEN_PERM:
803 /* follow indirections */
804 p = ((StgInd *)p)->indirectee;
809 return ((StgEvacuated *)p)->evacuee;
819 MarkRoot(StgClosure *root)
821 return evacuate(root);
824 static void addBlock(step *step)
826 bdescr *bd = allocBlock();
830 if (step->gen->no <= N) {
836 step->hp_bd->free = step->hp;
837 step->hp_bd->link = bd;
838 step->hp = bd->start;
839 step->hpLim = step->hp + BLOCK_SIZE_W;
845 static __inline__ StgClosure *
846 copy(StgClosure *src, nat size, step *step)
850 TICK_GC_WORDS_COPIED(size);
851 /* Find out where we're going, using the handy "to" pointer in
852 * the step of the source object. If it turns out we need to
853 * evacuate to an older generation, adjust it here (see comment
856 if (step->gen->no < evac_gen) {
857 step = &generations[evac_gen].steps[0];
860 /* chain a new block onto the to-space for the destination step if
863 if (step->hp + size >= step->hpLim) {
867 for(to = step->hp, from = (P_)src; size>0; --size) {
873 return (StgClosure *)dest;
876 /* Special version of copy() for when we only want to copy the info
877 * pointer of an object, but reserve some padding after it. This is
878 * used to optimise evacuation of BLACKHOLEs.
881 static __inline__ StgClosure *
882 copyPart(StgClosure *src, nat size_to_reserve, nat size_to_copy, step *step)
886 TICK_GC_WORDS_COPIED(size_to_copy);
887 if (step->gen->no < evac_gen) {
888 step = &generations[evac_gen].steps[0];
891 if (step->hp + size_to_reserve >= step->hpLim) {
895 for(to = step->hp, from = (P_)src; size_to_copy>0; --size_to_copy) {
900 step->hp += size_to_reserve;
901 return (StgClosure *)dest;
904 static __inline__ void
905 upd_evacuee(StgClosure *p, StgClosure *dest)
907 StgEvacuated *q = (StgEvacuated *)p;
909 SET_INFO(q,&EVACUATED_info);
913 /* -----------------------------------------------------------------------------
914 Evacuate a large object
916 This just consists of removing the object from the (doubly-linked)
917 large_alloc_list, and linking it on to the (singly-linked)
918 new_large_objects list, from where it will be scavenged later.
920 Convention: bd->evacuated is /= 0 for a large object that has been
921 evacuated, or 0 otherwise.
922 -------------------------------------------------------------------------- */
925 evacuate_large(StgPtr p, rtsBool mutable)
927 bdescr *bd = Bdescr(p);
930 /* should point to the beginning of the block */
931 ASSERT(((W_)p & BLOCK_MASK) == 0);
933 /* already evacuated? */
935 /* Don't forget to set the failed_to_evac flag if we didn't get
936 * the desired destination (see comments in evacuate()).
938 if (bd->gen->no < evac_gen) {
939 failed_to_evac = rtsTrue;
940 TICK_GC_FAILED_PROMOTION();
946 /* remove from large_object list */
948 bd->back->link = bd->link;
949 } else { /* first object in the list */
950 step->large_objects = bd->link;
953 bd->link->back = bd->back;
956 /* link it on to the evacuated large object list of the destination step
959 if (step->gen->no < evac_gen) {
960 step = &generations[evac_gen].steps[0];
965 bd->link = step->new_large_objects;
966 step->new_large_objects = bd;
970 recordMutable((StgMutClosure *)p);
974 /* -----------------------------------------------------------------------------
975 Adding a MUT_CONS to an older generation.
977 This is necessary from time to time when we end up with an
978 old-to-new generation pointer in a non-mutable object. We defer
979 the promotion until the next GC.
980 -------------------------------------------------------------------------- */
983 mkMutCons(StgClosure *ptr, generation *gen)
988 step = &gen->steps[0];
990 /* chain a new block onto the to-space for the destination step if
993 if (step->hp + sizeofW(StgIndOldGen) >= step->hpLim) {
997 q = (StgMutVar *)step->hp;
998 step->hp += sizeofW(StgMutVar);
1000 SET_HDR(q,&MUT_CONS_info,CCS_GC);
1002 recordOldToNewPtrs((StgMutClosure *)q);
1004 return (StgClosure *)q;
1007 /* -----------------------------------------------------------------------------
1010 This is called (eventually) for every live object in the system.
1012 The caller to evacuate specifies a desired generation in the
1013 evac_gen global variable. The following conditions apply to
1014 evacuating an object which resides in generation M when we're
1015 collecting up to generation N
1019 else evac to step->to
1021 if M < evac_gen evac to evac_gen, step 0
1023 if the object is already evacuated, then we check which generation
1026 if M >= evac_gen do nothing
1027 if M < evac_gen set failed_to_evac flag to indicate that we
1028 didn't manage to evacuate this object into evac_gen.
1030 -------------------------------------------------------------------------- */
1034 evacuate(StgClosure *q)
1039 const StgInfoTable *info;
1042 if (!LOOKS_LIKE_STATIC(q)) {
1044 if (bd->gen->no > N) {
1045 /* Can't evacuate this object, because it's in a generation
1046 * older than the ones we're collecting. Let's hope that it's
1047 * in evac_gen or older, or we will have to make an IND_OLDGEN object.
1049 if (bd->gen->no < evac_gen) {
1051 failed_to_evac = rtsTrue;
1052 TICK_GC_FAILED_PROMOTION();
1056 step = bd->step->to;
1059 /* make sure the info pointer is into text space */
1060 ASSERT(q && (LOOKS_LIKE_GHC_INFO(GET_INFO(q))
1061 || IS_HUGS_CONSTR_INFO(GET_INFO(q))));
1064 switch (info -> type) {
1067 to = copy(q,bco_sizeW(stgCast(StgBCO*,q)),step);
1072 ASSERT(q->header.info != &MUT_CONS_info);
1074 to = copy(q,sizeW_fromITBL(info),step);
1076 recordMutable((StgMutClosure *)to);
1080 stable_ptr_table[((StgStableName *)q)->sn].keep = rtsTrue;
1081 to = copy(q,sizeofW(StgStableName),step);
1089 to = copy(q,sizeofW(StgHeader)+1,step);
1093 case THUNK_1_0: /* here because of MIN_UPD_SIZE */
1104 to = copy(q,sizeofW(StgHeader)+2,step);
1112 case IND_OLDGEN_PERM:
1117 to = copy(q,sizeW_fromITBL(info),step);
1123 to = copyPart(q,BLACKHOLE_sizeW(),sizeofW(StgHeader),step);
1128 to = copy(q,BLACKHOLE_sizeW(),step);
1130 recordMutable((StgMutClosure *)to);
1133 case THUNK_SELECTOR:
1135 const StgInfoTable* selectee_info;
1136 StgClosure* selectee = ((StgSelector*)q)->selectee;
1139 selectee_info = get_itbl(selectee);
1140 switch (selectee_info->type) {
1149 StgNat32 offset = info->layout.selector_offset;
1151 /* check that the size is in range */
1153 (StgNat32)(selectee_info->layout.payload.ptrs +
1154 selectee_info->layout.payload.nptrs));
1156 /* perform the selection! */
1157 q = selectee->payload[offset];
1159 /* if we're already in to-space, there's no need to continue
1160 * with the evacuation, just update the source address with
1161 * a pointer to the (evacuated) constructor field.
1163 if (IS_USER_PTR(q)) {
1164 bdescr *bd = Bdescr((P_)q);
1165 if (bd->evacuated) {
1166 if (bd->gen->no < evac_gen) {
1167 failed_to_evac = rtsTrue;
1168 TICK_GC_FAILED_PROMOTION();
1174 /* otherwise, carry on and evacuate this constructor field,
1175 * (but not the constructor itself)
1184 case IND_OLDGEN_PERM:
1185 selectee = stgCast(StgInd *,selectee)->indirectee;
1189 selectee = stgCast(StgCAF *,selectee)->value;
1193 selectee = stgCast(StgEvacuated*,selectee)->evacuee;
1203 case THUNK_SELECTOR:
1204 /* aargh - do recursively???? */
1209 /* not evaluated yet */
1213 barf("evacuate: THUNK_SELECTOR: strange selectee");
1216 to = copy(q,THUNK_SELECTOR_sizeW(),step);
1222 /* follow chains of indirections, don't evacuate them */
1223 q = ((StgInd*)q)->indirectee;
1226 /* ToDo: optimise STATIC_LINK for known cases.
1227 - FUN_STATIC : payload[0]
1228 - THUNK_STATIC : payload[1]
1229 - IND_STATIC : payload[1]
1233 if (info->srt_len == 0) { /* small optimisation */
1239 /* don't want to evacuate these, but we do want to follow pointers
1240 * from SRTs - see scavenge_static.
1243 /* put the object on the static list, if necessary.
1245 if (major_gc && STATIC_LINK(info,(StgClosure *)q) == NULL) {
1246 STATIC_LINK(info,(StgClosure *)q) = static_objects;
1247 static_objects = (StgClosure *)q;
1251 case CONSTR_INTLIKE:
1252 case CONSTR_CHARLIKE:
1253 case CONSTR_NOCAF_STATIC:
1254 /* no need to put these on the static linked list, they don't need
1269 /* shouldn't see these */
1270 barf("evacuate: stack frame\n");
1274 /* these are special - the payload is a copy of a chunk of stack,
1276 to = copy(q,pap_sizeW(stgCast(StgPAP*,q)),step);
1281 /* Already evacuated, just return the forwarding address.
1282 * HOWEVER: if the requested destination generation (evac_gen) is
1283 * older than the actual generation (because the object was
1284 * already evacuated to a younger generation) then we have to
1285 * set the failed_to_evac flag to indicate that we couldn't
1286 * manage to promote the object to the desired generation.
1288 if (evac_gen > 0) { /* optimisation */
1289 StgClosure *p = ((StgEvacuated*)q)->evacuee;
1290 if (Bdescr((P_)p)->gen->no < evac_gen) {
1291 /* fprintf(stderr,"evac failed!\n");*/
1292 failed_to_evac = rtsTrue;
1293 TICK_GC_FAILED_PROMOTION();
1296 return ((StgEvacuated*)q)->evacuee;
1300 nat size = arr_words_sizeW(stgCast(StgArrWords*,q));
1302 if (size >= LARGE_OBJECT_THRESHOLD/sizeof(W_)) {
1303 evacuate_large((P_)q, rtsFalse);
1306 /* just copy the block */
1307 to = copy(q,size,step);
1314 case MUT_ARR_PTRS_FROZEN:
1316 nat size = mut_arr_ptrs_sizeW(stgCast(StgMutArrPtrs*,q));
1318 if (size >= LARGE_OBJECT_THRESHOLD/sizeof(W_)) {
1319 evacuate_large((P_)q, info->type == MUT_ARR_PTRS);
1322 /* just copy the block */
1323 to = copy(q,size,step);
1325 if (info->type == MUT_ARR_PTRS) {
1326 recordMutable((StgMutClosure *)to);
1334 StgTSO *tso = stgCast(StgTSO *,q);
1335 nat size = tso_sizeW(tso);
1338 /* Large TSOs don't get moved, so no relocation is required.
1340 if (size >= LARGE_OBJECT_THRESHOLD/sizeof(W_)) {
1341 evacuate_large((P_)q, rtsTrue);
1344 /* To evacuate a small TSO, we need to relocate the update frame
1348 StgTSO *new_tso = (StgTSO *)copy((StgClosure *)tso,tso_sizeW(tso),step);
1350 diff = (StgPtr)new_tso - (StgPtr)tso; /* In *words* */
1352 /* relocate the stack pointers... */
1353 new_tso->su = (StgUpdateFrame *) ((StgPtr)new_tso->su + diff);
1354 new_tso->sp = (StgPtr)new_tso->sp + diff;
1355 new_tso->splim = (StgPtr)new_tso->splim + diff;
1357 relocate_TSO(tso, new_tso);
1358 upd_evacuee(q,(StgClosure *)new_tso);
1360 recordMutable((StgMutClosure *)new_tso);
1361 return (StgClosure *)new_tso;
1367 fprintf(stderr,"evacuate: unimplemented/strange closure type\n");
1371 barf("evacuate: strange closure type");
1377 /* -----------------------------------------------------------------------------
1378 relocate_TSO is called just after a TSO has been copied from src to
1379 dest. It adjusts the update frame list for the new location.
1380 -------------------------------------------------------------------------- */
1383 relocate_TSO(StgTSO *src, StgTSO *dest)
1390 diff = (StgPtr)dest->sp - (StgPtr)src->sp; /* In *words* */
1394 while ((P_)su < dest->stack + dest->stack_size) {
1395 switch (get_itbl(su)->type) {
1397 /* GCC actually manages to common up these three cases! */
1400 su->link = (StgUpdateFrame *) ((StgPtr)su->link + diff);
1405 cf = (StgCatchFrame *)su;
1406 cf->link = (StgUpdateFrame *) ((StgPtr)cf->link + diff);
1411 sf = (StgSeqFrame *)su;
1412 sf->link = (StgUpdateFrame *) ((StgPtr)sf->link + diff);
1421 barf("relocate_TSO");
1430 scavenge_srt(const StgInfoTable *info)
1432 StgClosure **srt, **srt_end;
1434 /* evacuate the SRT. If srt_len is zero, then there isn't an
1435 * srt field in the info table. That's ok, because we'll
1436 * never dereference it.
1438 srt = stgCast(StgClosure **,info->srt);
1439 srt_end = srt + info->srt_len;
1440 for (; srt < srt_end; srt++) {
1445 /* -----------------------------------------------------------------------------
1446 Scavenge a given step until there are no more objects in this step
1449 evac_gen is set by the caller to be either zero (for a step in a
1450 generation < N) or G where G is the generation of the step being
1453 We sometimes temporarily change evac_gen back to zero if we're
1454 scavenging a mutable object where early promotion isn't such a good
1456 -------------------------------------------------------------------------- */
1460 scavenge(step *step)
1463 const StgInfoTable *info;
1465 nat saved_evac_gen = evac_gen; /* used for temporarily changing evac_gen */
1470 failed_to_evac = rtsFalse;
1472 /* scavenge phase - standard breadth-first scavenging of the
1476 while (bd != step->hp_bd || p < step->hp) {
1478 /* If we're at the end of this block, move on to the next block */
1479 if (bd != step->hp_bd && p == bd->free) {
1485 q = p; /* save ptr to object */
1487 ASSERT(p && (LOOKS_LIKE_GHC_INFO(GET_INFO((StgClosure *)p))
1488 || IS_HUGS_CONSTR_INFO(GET_INFO((StgClosure *)p))));
1490 info = get_itbl((StgClosure *)p);
1491 switch (info -> type) {
1495 StgBCO* bco = stgCast(StgBCO*,p);
1497 for (i = 0; i < bco->n_ptrs; i++) {
1498 bcoConstCPtr(bco,i) = evacuate(bcoConstCPtr(bco,i));
1500 p += bco_sizeW(bco);
1505 /* treat MVars specially, because we don't want to evacuate the
1506 * mut_link field in the middle of the closure.
1509 StgMVar *mvar = ((StgMVar *)p);
1511 (StgClosure *)mvar->head = evacuate((StgClosure *)mvar->head);
1512 (StgClosure *)mvar->tail = evacuate((StgClosure *)mvar->tail);
1513 (StgClosure *)mvar->value = evacuate((StgClosure *)mvar->value);
1514 p += sizeofW(StgMVar);
1515 evac_gen = saved_evac_gen;
1523 ((StgClosure *)p)->payload[1] = evacuate(((StgClosure *)p)->payload[1]);
1524 ((StgClosure *)p)->payload[0] = evacuate(((StgClosure *)p)->payload[0]);
1525 p += sizeofW(StgHeader) + 2;
1530 ((StgClosure *)p)->payload[0] = evacuate(((StgClosure *)p)->payload[0]);
1531 p += sizeofW(StgHeader) + 2; /* MIN_UPD_SIZE */
1537 ((StgClosure *)p)->payload[0] = evacuate(((StgClosure *)p)->payload[0]);
1538 p += sizeofW(StgHeader) + 1;
1543 p += sizeofW(StgHeader) + 2; /* MIN_UPD_SIZE */
1549 p += sizeofW(StgHeader) + 1;
1556 p += sizeofW(StgHeader) + 2;
1563 ((StgClosure *)p)->payload[0] = evacuate(((StgClosure *)p)->payload[0]);
1564 p += sizeofW(StgHeader) + 2;
1577 case IND_OLDGEN_PERM:
1583 end = (P_)((StgClosure *)p)->payload + info->layout.payload.ptrs;
1584 for (p = (P_)((StgClosure *)p)->payload; p < end; p++) {
1585 (StgClosure *)*p = evacuate((StgClosure *)*p);
1587 p += info->layout.payload.nptrs;
1592 /* ignore MUT_CONSs */
1593 if (((StgMutVar *)p)->header.info != &MUT_CONS_info) {
1595 ((StgMutVar *)p)->var = evacuate(((StgMutVar *)p)->var);
1596 evac_gen = saved_evac_gen;
1598 p += sizeofW(StgMutVar);
1603 p += BLACKHOLE_sizeW();
1608 StgBlockingQueue *bh = (StgBlockingQueue *)p;
1609 (StgClosure *)bh->blocking_queue =
1610 evacuate((StgClosure *)bh->blocking_queue);
1611 if (failed_to_evac) {
1612 failed_to_evac = rtsFalse;
1613 recordMutable((StgMutClosure *)bh);
1615 p += BLACKHOLE_sizeW();
1619 case THUNK_SELECTOR:
1621 StgSelector *s = (StgSelector *)p;
1622 s->selectee = evacuate(s->selectee);
1623 p += THUNK_SELECTOR_sizeW();
1629 barf("scavenge:IND???\n");
1631 case CONSTR_INTLIKE:
1632 case CONSTR_CHARLIKE:
1634 case CONSTR_NOCAF_STATIC:
1638 /* Shouldn't see a static object here. */
1639 barf("scavenge: STATIC object\n");
1651 /* Shouldn't see stack frames here. */
1652 barf("scavenge: stack frame\n");
1654 case AP_UPD: /* same as PAPs */
1656 /* Treat a PAP just like a section of stack, not forgetting to
1657 * evacuate the function pointer too...
1660 StgPAP* pap = stgCast(StgPAP*,p);
1662 pap->fun = evacuate(pap->fun);
1663 scavenge_stack((P_)pap->payload, (P_)pap->payload + pap->n_args);
1664 p += pap_sizeW(pap);
1669 /* nothing to follow */
1670 p += arr_words_sizeW(stgCast(StgArrWords*,p));
1674 /* follow everything */
1678 evac_gen = 0; /* repeatedly mutable */
1679 next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
1680 for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
1681 (StgClosure *)*p = evacuate((StgClosure *)*p);
1683 evac_gen = saved_evac_gen;
1687 case MUT_ARR_PTRS_FROZEN:
1688 /* follow everything */
1690 StgPtr start = p, next;
1692 next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
1693 for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
1694 (StgClosure *)*p = evacuate((StgClosure *)*p);
1696 if (failed_to_evac) {
1697 /* we can do this easier... */
1698 recordMutable((StgMutClosure *)start);
1699 failed_to_evac = rtsFalse;
1710 /* chase the link field for any TSOs on the same queue */
1711 (StgClosure *)tso->link = evacuate((StgClosure *)tso->link);
1712 /* scavenge this thread's stack */
1713 scavenge_stack(tso->sp, &(tso->stack[tso->stack_size]));
1714 evac_gen = saved_evac_gen;
1715 p += tso_sizeW(tso);
1722 barf("scavenge: unimplemented/strange closure type\n");
1728 /* If we didn't manage to promote all the objects pointed to by
1729 * the current object, then we have to designate this object as
1730 * mutable (because it contains old-to-new generation pointers).
1732 if (failed_to_evac) {
1733 mkMutCons((StgClosure *)q, &generations[evac_gen]);
1734 failed_to_evac = rtsFalse;
1742 /* -----------------------------------------------------------------------------
1743 Scavenge one object.
1745 This is used for objects that are temporarily marked as mutable
1746 because they contain old-to-new generation pointers. Only certain
1747 objects can have this property.
1748 -------------------------------------------------------------------------- */
1750 scavenge_one(StgClosure *p)
1755 ASSERT(p && (LOOKS_LIKE_GHC_INFO(GET_INFO(p))
1756 || IS_HUGS_CONSTR_INFO(GET_INFO(p))));
1760 switch (info -> type) {
1763 case FUN_1_0: /* hardly worth specialising these guys */
1783 case IND_OLDGEN_PERM:
1789 end = (P_)p->payload + info->layout.payload.ptrs;
1790 for (q = (P_)p->payload; q < end; q++) {
1791 (StgClosure *)*q = evacuate((StgClosure *)*q);
1800 case THUNK_SELECTOR:
1802 StgSelector *s = (StgSelector *)p;
1803 s->selectee = evacuate(s->selectee);
1807 case AP_UPD: /* same as PAPs */
1809 /* Treat a PAP just like a section of stack, not forgetting to
1810 * evacuate the function pointer too...
1813 StgPAP* pap = (StgPAP *)p;
1815 pap->fun = evacuate(pap->fun);
1816 scavenge_stack((P_)pap->payload, (P_)pap->payload + pap->n_args);
1821 /* This might happen if for instance a MUT_CONS was pointing to a
1822 * THUNK which has since been updated. The IND_OLDGEN will
1823 * be on the mutable list anyway, so we don't need to do anything
1829 barf("scavenge_one: strange object");
1832 no_luck = failed_to_evac;
1833 failed_to_evac = rtsFalse;
1838 /* -----------------------------------------------------------------------------
1839 Scavenging mutable lists.
1841 We treat the mutable list of each generation > N (i.e. all the
1842 generations older than the one being collected) as roots. We also
1843 remove non-mutable objects from the mutable list at this point.
1844 -------------------------------------------------------------------------- */
1847 scavenge_mut_once_list(generation *gen)
1850 StgMutClosure *p, *next, *new_list;
1852 p = gen->mut_once_list;
1853 new_list = END_MUT_LIST;
1857 failed_to_evac = rtsFalse;
1859 for (; p != END_MUT_LIST; p = next, next = p->mut_link) {
1861 /* make sure the info pointer is into text space */
1862 ASSERT(p && (LOOKS_LIKE_GHC_INFO(GET_INFO(p))
1863 || IS_HUGS_CONSTR_INFO(GET_INFO(p))));
1866 switch(info->type) {
1869 case IND_OLDGEN_PERM:
1871 /* Try to pull the indirectee into this generation, so we can
1872 * remove the indirection from the mutable list.
1874 ((StgIndOldGen *)p)->indirectee =
1875 evacuate(((StgIndOldGen *)p)->indirectee);
1878 /* Debugging code to print out the size of the thing we just
1882 StgPtr start = gen->steps[0].scan;
1883 bdescr *start_bd = gen->steps[0].scan_bd;
1885 scavenge(&gen->steps[0]);
1886 if (start_bd != gen->steps[0].scan_bd) {
1887 size += (P_)BLOCK_ROUND_UP(start) - start;
1888 start_bd = start_bd->link;
1889 while (start_bd != gen->steps[0].scan_bd) {
1890 size += BLOCK_SIZE_W;
1891 start_bd = start_bd->link;
1893 size += gen->steps[0].scan -
1894 (P_)BLOCK_ROUND_DOWN(gen->steps[0].scan);
1896 size = gen->steps[0].scan - start;
1898 fprintf(stderr,"evac IND_OLDGEN: %d bytes\n", size * sizeof(W_));
1902 /* failed_to_evac might happen if we've got more than two
1903 * generations, we're collecting only generation 0, the
1904 * indirection resides in generation 2 and the indirectee is
1907 if (failed_to_evac) {
1908 failed_to_evac = rtsFalse;
1909 p->mut_link = new_list;
1912 /* the mut_link field of an IND_STATIC is overloaded as the
1913 * static link field too (it just so happens that we don't need
1914 * both at the same time), so we need to NULL it out when
1915 * removing this object from the mutable list because the static
1916 * link fields are all assumed to be NULL before doing a major
1924 /* MUT_CONS is a kind of MUT_VAR, except it that we try to remove
1925 * it from the mutable list if possible by promoting whatever it
1928 ASSERT(p->header.info == &MUT_CONS_info);
1929 if (scavenge_one(((StgMutVar *)p)->var) == rtsTrue) {
1930 /* didn't manage to promote everything, so put the
1931 * MUT_CONS back on the list.
1933 p->mut_link = new_list;
1939 /* shouldn't have anything else on the mutables list */
1940 barf("scavenge_mut_once_list: strange object?");
1944 gen->mut_once_list = new_list;
1949 scavenge_mutable_list(generation *gen)
1952 StgMutClosure *p, *next;
1954 p = gen->saved_mut_list;
1958 failed_to_evac = rtsFalse;
1960 for (; p != END_MUT_LIST; p = next, next = p->mut_link) {
1962 /* make sure the info pointer is into text space */
1963 ASSERT(p && (LOOKS_LIKE_GHC_INFO(GET_INFO(p))
1964 || IS_HUGS_CONSTR_INFO(GET_INFO(p))));
1967 switch(info->type) {
1969 case MUT_ARR_PTRS_FROZEN:
1970 /* remove this guy from the mutable list, but follow the ptrs
1971 * anyway (and make sure they get promoted to this gen).
1976 end = (P_)p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
1978 for (q = (P_)((StgMutArrPtrs *)p)->payload; q < end; q++) {
1979 (StgClosure *)*q = evacuate((StgClosure *)*q);
1983 if (failed_to_evac) {
1984 failed_to_evac = rtsFalse;
1985 p->mut_link = gen->mut_list;
1992 /* follow everything */
1993 p->mut_link = gen->mut_list;
1998 end = (P_)p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
1999 for (q = (P_)((StgMutArrPtrs *)p)->payload; q < end; q++) {
2000 (StgClosure *)*q = evacuate((StgClosure *)*q);
2006 /* MUT_CONS is a kind of MUT_VAR, except that we try to remove
2007 * it from the mutable list if possible by promoting whatever it
2010 ASSERT(p->header.info != &MUT_CONS_info);
2011 ((StgMutVar *)p)->var = evacuate(((StgMutVar *)p)->var);
2012 p->mut_link = gen->mut_list;
2018 StgMVar *mvar = (StgMVar *)p;
2019 (StgClosure *)mvar->head = evacuate((StgClosure *)mvar->head);
2020 (StgClosure *)mvar->tail = evacuate((StgClosure *)mvar->tail);
2021 (StgClosure *)mvar->value = evacuate((StgClosure *)mvar->value);
2022 p->mut_link = gen->mut_list;
2028 /* follow ptrs and remove this from the mutable list */
2030 StgTSO *tso = (StgTSO *)p;
2032 /* Don't bother scavenging if this thread is dead
2034 if (!(tso->whatNext == ThreadComplete ||
2035 tso->whatNext == ThreadKilled)) {
2036 /* Don't need to chase the link field for any TSOs on the
2037 * same queue. Just scavenge this thread's stack
2039 scavenge_stack(tso->sp, &(tso->stack[tso->stack_size]));
2042 /* Don't take this TSO off the mutable list - it might still
2043 * point to some younger objects (because we set evac_gen to 0
2046 tso->mut_link = gen->mut_list;
2047 gen->mut_list = (StgMutClosure *)tso;
2053 StgBlockingQueue *bh = (StgBlockingQueue *)p;
2054 (StgClosure *)bh->blocking_queue =
2055 evacuate((StgClosure *)bh->blocking_queue);
2056 p->mut_link = gen->mut_list;
2062 /* shouldn't have anything else on the mutables list */
2063 barf("scavenge_mut_list: strange object?");
2069 scavenge_static(void)
2071 StgClosure* p = static_objects;
2072 const StgInfoTable *info;
2074 /* Always evacuate straight to the oldest generation for static
2076 evac_gen = oldest_gen->no;
2078 /* keep going until we've scavenged all the objects on the linked
2080 while (p != END_OF_STATIC_LIST) {
2084 /* make sure the info pointer is into text space */
2085 ASSERT(p && (LOOKS_LIKE_GHC_INFO(GET_INFO(p))
2086 || IS_HUGS_CONSTR_INFO(GET_INFO(p))));
2088 /* Take this object *off* the static_objects list,
2089 * and put it on the scavenged_static_objects list.
2091 static_objects = STATIC_LINK(info,p);
2092 STATIC_LINK(info,p) = scavenged_static_objects;
2093 scavenged_static_objects = p;
2095 switch (info -> type) {
2099 StgInd *ind = (StgInd *)p;
2100 ind->indirectee = evacuate(ind->indirectee);
2102 /* might fail to evacuate it, in which case we have to pop it
2103 * back on the mutable list (and take it off the
2104 * scavenged_static list because the static link and mut link
2105 * pointers are one and the same).
2107 if (failed_to_evac) {
2108 failed_to_evac = rtsFalse;
2109 scavenged_static_objects = STATIC_LINK(info,p);
2110 ((StgMutClosure *)ind)->mut_link = oldest_gen->mut_once_list;
2111 oldest_gen->mut_once_list = (StgMutClosure *)ind;
2125 next = (P_)p->payload + info->layout.payload.ptrs;
2126 /* evacuate the pointers */
2127 for (q = (P_)p->payload; q < next; q++) {
2128 (StgClosure *)*q = evacuate((StgClosure *)*q);
2134 barf("scavenge_static");
2137 ASSERT(failed_to_evac == rtsFalse);
2139 /* get the next static object from the list. Remeber, there might
2140 * be more stuff on this list now that we've done some evacuating!
2141 * (static_objects is a global)
2147 /* -----------------------------------------------------------------------------
2148 scavenge_stack walks over a section of stack and evacuates all the
2149 objects pointed to by it. We can use the same code for walking
2150 PAPs, since these are just sections of copied stack.
2151 -------------------------------------------------------------------------- */
2154 scavenge_stack(StgPtr p, StgPtr stack_end)
2157 const StgInfoTable* info;
2161 * Each time around this loop, we are looking at a chunk of stack
2162 * that starts with either a pending argument section or an
2163 * activation record.
2166 while (p < stack_end) {
2167 q = *stgCast(StgPtr*,p);
2169 /* If we've got a tag, skip over that many words on the stack */
2170 if (IS_ARG_TAG(stgCast(StgWord,q))) {
2175 /* Is q a pointer to a closure?
2177 if (! LOOKS_LIKE_GHC_INFO(q)) {
2180 if (LOOKS_LIKE_STATIC(q)) { /* Is it a static closure? */
2181 ASSERT(closure_STATIC(stgCast(StgClosure*,q)));
2183 /* otherwise, must be a pointer into the allocation space.
2187 (StgClosure *)*p = evacuate((StgClosure *)q);
2193 * Otherwise, q must be the info pointer of an activation
2194 * record. All activation records have 'bitmap' style layout
2197 info = get_itbl(stgCast(StgClosure*,p));
2199 switch (info->type) {
2201 /* Dynamic bitmap: the mask is stored on the stack */
2203 bitmap = stgCast(StgRetDyn*,p)->liveness;
2204 p = &payloadWord(stgCast(StgRetDyn*,p),0);
2207 /* probably a slow-entry point return address: */
2213 /* Specialised code for update frames, since they're so common.
2214 * We *know* the updatee points to a BLACKHOLE, CAF_BLACKHOLE,
2215 * or BLACKHOLE_BQ, so just inline the code to evacuate it here.
2219 StgUpdateFrame *frame = (StgUpdateFrame *)p;
2221 StgClosureType type = get_itbl(frame->updatee)->type;
2223 p += sizeofW(StgUpdateFrame);
2224 if (type == EVACUATED) {
2225 frame->updatee = evacuate(frame->updatee);
2228 bdescr *bd = Bdescr((P_)frame->updatee);
2230 if (bd->gen->no > N) {
2231 if (bd->gen->no < evac_gen) {
2232 failed_to_evac = rtsTrue;
2236 step = bd->step->to;
2240 to = copyPart(frame->updatee, BLACKHOLE_sizeW(),
2241 sizeofW(StgHeader), step);
2242 upd_evacuee(frame->updatee,to);
2243 frame->updatee = to;
2246 to = copy(frame->updatee, BLACKHOLE_sizeW(), step);
2247 upd_evacuee(frame->updatee,to);
2248 frame->updatee = to;
2249 recordMutable((StgMutClosure *)to);
2252 barf("scavenge_stack: UPDATE_FRAME updatee");
2257 /* small bitmap (< 32 entries, or 64 on a 64-bit machine) */
2264 bitmap = info->layout.bitmap;
2267 while (bitmap != 0) {
2268 if ((bitmap & 1) == 0) {
2269 (StgClosure *)*p = evacuate((StgClosure *)*p);
2272 bitmap = bitmap >> 1;
2279 /* large bitmap (> 32 entries) */
2284 StgLargeBitmap *large_bitmap;
2287 large_bitmap = info->layout.large_bitmap;
2290 for (i=0; i<large_bitmap->size; i++) {
2291 bitmap = large_bitmap->bitmap[i];
2292 q = p + sizeof(W_) * 8;
2293 while (bitmap != 0) {
2294 if ((bitmap & 1) == 0) {
2295 (StgClosure *)*p = evacuate((StgClosure *)*p);
2298 bitmap = bitmap >> 1;
2300 if (i+1 < large_bitmap->size) {
2302 (StgClosure *)*p = evacuate((StgClosure *)*p);
2308 /* and don't forget to follow the SRT */
2313 barf("scavenge_stack: weird activation record found on stack.\n");
2318 /*-----------------------------------------------------------------------------
2319 scavenge the large object list.
2321 evac_gen set by caller; similar games played with evac_gen as with
2322 scavenge() - see comment at the top of scavenge(). Most large
2323 objects are (repeatedly) mutable, so most of the time evac_gen will
2325 --------------------------------------------------------------------------- */
2328 scavenge_large(step *step)
2332 const StgInfoTable* info;
2333 nat saved_evac_gen = evac_gen; /* used for temporarily changing evac_gen */
2335 evac_gen = 0; /* most objects are mutable */
2336 bd = step->new_large_objects;
2338 for (; bd != NULL; bd = step->new_large_objects) {
2340 /* take this object *off* the large objects list and put it on
2341 * the scavenged large objects list. This is so that we can
2342 * treat new_large_objects as a stack and push new objects on
2343 * the front when evacuating.
2345 step->new_large_objects = bd->link;
2346 dbl_link_onto(bd, &step->scavenged_large_objects);
2349 info = get_itbl(stgCast(StgClosure*,p));
2351 switch (info->type) {
2353 /* only certain objects can be "large"... */
2356 /* nothing to follow */
2360 /* follow everything */
2364 next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
2365 for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
2366 (StgClosure *)*p = evacuate((StgClosure *)*p);
2371 case MUT_ARR_PTRS_FROZEN:
2372 /* follow everything */
2374 StgPtr start = p, next;
2376 evac_gen = saved_evac_gen; /* not really mutable */
2377 next = p + mut_arr_ptrs_sizeW((StgMutArrPtrs*)p);
2378 for (p = (P_)((StgMutArrPtrs *)p)->payload; p < next; p++) {
2379 (StgClosure *)*p = evacuate((StgClosure *)*p);
2382 if (failed_to_evac) {
2383 recordMutable((StgMutClosure *)start);
2390 StgBCO* bco = stgCast(StgBCO*,p);
2392 evac_gen = saved_evac_gen;
2393 for (i = 0; i < bco->n_ptrs; i++) {
2394 bcoConstCPtr(bco,i) = evacuate(bcoConstCPtr(bco,i));
2405 /* chase the link field for any TSOs on the same queue */
2406 (StgClosure *)tso->link = evacuate((StgClosure *)tso->link);
2407 /* scavenge this thread's stack */
2408 scavenge_stack(tso->sp, &(tso->stack[tso->stack_size]));
2413 barf("scavenge_large: unknown/strange object");
2419 zeroStaticObjectList(StgClosure* first_static)
2423 const StgInfoTable *info;
2425 for (p = first_static; p != END_OF_STATIC_LIST; p = link) {
2427 link = STATIC_LINK(info, p);
2428 STATIC_LINK(info,p) = NULL;
2432 /* This function is only needed because we share the mutable link
2433 * field with the static link field in an IND_STATIC, so we have to
2434 * zero the mut_link field before doing a major GC, which needs the
2435 * static link field.
2437 * It doesn't do any harm to zero all the mutable link fields on the
2441 zeroMutableList(StgMutClosure *first)
2443 StgMutClosure *next, *c;
2445 for (c = first; c != END_MUT_LIST; c = next) {
2451 /* -----------------------------------------------------------------------------
2453 -------------------------------------------------------------------------- */
2455 void RevertCAFs(void)
2457 while (enteredCAFs != END_CAF_LIST) {
2458 StgCAF* caf = enteredCAFs;
2460 enteredCAFs = caf->link;
2461 ASSERT(get_itbl(caf)->type == CAF_ENTERED);
2462 SET_INFO(caf,&CAF_UNENTERED_info);
2463 caf->value = stgCast(StgClosure*,0xdeadbeef);
2464 caf->link = stgCast(StgCAF*,0xdeadbeef);
2468 void revertDeadCAFs(void)
2470 StgCAF* caf = enteredCAFs;
2471 enteredCAFs = END_CAF_LIST;
2472 while (caf != END_CAF_LIST) {
2473 StgCAF* next = caf->link;
2475 switch(GET_INFO(caf)->type) {
2478 /* This object has been evacuated, it must be live. */
2479 StgCAF* new = stgCast(StgCAF*,stgCast(StgEvacuated*,caf)->evacuee);
2480 new->link = enteredCAFs;
2486 SET_INFO(caf,&CAF_UNENTERED_info);
2487 caf->value = stgCast(StgClosure*,0xdeadbeef);
2488 caf->link = stgCast(StgCAF*,0xdeadbeef);
2492 barf("revertDeadCAFs: enteredCAFs list corrupted");
2498 /* -----------------------------------------------------------------------------
2499 Sanity code for CAF garbage collection.
2501 With DEBUG turned on, we manage a CAF list in addition to the SRT
2502 mechanism. After GC, we run down the CAF list and blackhole any
2503 CAFs which have been garbage collected. This means we get an error
2504 whenever the program tries to enter a garbage collected CAF.
2506 Any garbage collected CAFs are taken off the CAF list at the same
2508 -------------------------------------------------------------------------- */
2516 const StgInfoTable *info;
2527 ASSERT(info->type == IND_STATIC);
2529 if (STATIC_LINK(info,p) == NULL) {
2530 IF_DEBUG(gccafs, fprintf(stderr, "CAF gc'd at 0x%04x\n", (int)p));
2532 SET_INFO(p,&BLACKHOLE_info);
2533 p = STATIC_LINK2(info,p);
2537 pp = &STATIC_LINK2(info,p);
2544 /* fprintf(stderr, "%d CAFs live\n", i); */
2548 /* -----------------------------------------------------------------------------
2551 Whenever a thread returns to the scheduler after possibly doing
2552 some work, we have to run down the stack and black-hole all the
2553 closures referred to by update frames.
2554 -------------------------------------------------------------------------- */
2557 threadLazyBlackHole(StgTSO *tso)
2559 StgUpdateFrame *update_frame;
2560 StgBlockingQueue *bh;
2563 stack_end = &tso->stack[tso->stack_size];
2564 update_frame = tso->su;
2567 switch (get_itbl(update_frame)->type) {
2570 update_frame = stgCast(StgCatchFrame*,update_frame)->link;
2574 bh = (StgBlockingQueue *)update_frame->updatee;
2576 /* if the thunk is already blackholed, it means we've also
2577 * already blackholed the rest of the thunks on this stack,
2578 * so we can stop early.
2580 * The blackhole made for a CAF is a CAF_BLACKHOLE, so they
2581 * don't interfere with this optimisation.
2583 if (bh->header.info == &BLACKHOLE_info) {
2587 if (bh->header.info != &BLACKHOLE_BQ_info &&
2588 bh->header.info != &CAF_BLACKHOLE_info) {
2589 SET_INFO(bh,&BLACKHOLE_info);
2592 update_frame = update_frame->link;
2596 update_frame = stgCast(StgSeqFrame*,update_frame)->link;
2602 barf("threadPaused");
2607 /* -----------------------------------------------------------------------------
2610 * Code largely pinched from old RTS, then hacked to bits. We also do
2611 * lazy black holing here.
2613 * -------------------------------------------------------------------------- */
2616 threadSqueezeStack(StgTSO *tso)
2618 lnat displacement = 0;
2619 StgUpdateFrame *frame;
2620 StgUpdateFrame *next_frame; /* Temporally next */
2621 StgUpdateFrame *prev_frame; /* Temporally previous */
2623 rtsBool prev_was_update_frame;
2625 bottom = &(tso->stack[tso->stack_size]);
2628 /* There must be at least one frame, namely the STOP_FRAME.
2630 ASSERT((P_)frame < bottom);
2632 /* Walk down the stack, reversing the links between frames so that
2633 * we can walk back up as we squeeze from the bottom. Note that
2634 * next_frame and prev_frame refer to next and previous as they were
2635 * added to the stack, rather than the way we see them in this
2636 * walk. (It makes the next loop less confusing.)
2638 * Stop if we find an update frame pointing to a black hole
2639 * (see comment in threadLazyBlackHole()).
2643 while ((P_)frame < bottom - 1) { /* bottom - 1 is the STOP_FRAME */
2644 prev_frame = frame->link;
2645 frame->link = next_frame;
2648 if (get_itbl(frame)->type == UPDATE_FRAME
2649 && frame->updatee->header.info == &BLACKHOLE_info) {
2654 /* Now, we're at the bottom. Frame points to the lowest update
2655 * frame on the stack, and its link actually points to the frame
2656 * above. We have to walk back up the stack, squeezing out empty
2657 * update frames and turning the pointers back around on the way
2660 * The bottom-most frame (the STOP_FRAME) has not been altered, and
2661 * we never want to eliminate it anyway. Just walk one step up
2662 * before starting to squeeze. When you get to the topmost frame,
2663 * remember that there are still some words above it that might have
2670 prev_was_update_frame = (get_itbl(prev_frame)->type == UPDATE_FRAME);
2673 * Loop through all of the frames (everything except the very
2674 * bottom). Things are complicated by the fact that we have
2675 * CATCH_FRAMEs and SEQ_FRAMEs interspersed with the update frames.
2676 * We can only squeeze when there are two consecutive UPDATE_FRAMEs.
2678 while (frame != NULL) {
2680 StgPtr frame_bottom = (P_)frame + sizeofW(StgUpdateFrame);
2681 rtsBool is_update_frame;
2683 next_frame = frame->link;
2684 is_update_frame = (get_itbl(frame)->type == UPDATE_FRAME);
2687 * 1. both the previous and current frame are update frames
2688 * 2. the current frame is empty
2690 if (prev_was_update_frame && is_update_frame &&
2691 (P_)prev_frame == frame_bottom + displacement) {
2693 /* Now squeeze out the current frame */
2694 StgClosure *updatee_keep = prev_frame->updatee;
2695 StgClosure *updatee_bypass = frame->updatee;
2698 fprintf(stderr, "squeezing frame at %p\n", frame);
2701 /* Deal with blocking queues. If both updatees have blocked
2702 * threads, then we should merge the queues into the update
2703 * frame that we're keeping.
2705 * Alternatively, we could just wake them up: they'll just go
2706 * straight to sleep on the proper blackhole! This is less code
2707 * and probably less bug prone, although it's probably much
2710 #if 0 /* do it properly... */
2711 if (GET_INFO(updatee_bypass) == BLACKHOLE_BQ_info) {
2712 /* Sigh. It has one. Don't lose those threads! */
2713 if (GET_INFO(updatee_keep) == BLACKHOLE_BQ_info) {
2714 /* Urgh. Two queues. Merge them. */
2715 P_ keep_tso = ((StgBlockingQueue *)updatee_keep)->blocking_queue;
2717 while (keep_tso->link != END_TSO_QUEUE) {
2718 keep_tso = keep_tso->link;
2720 keep_tso->link = ((StgBlockingQueue *)updatee_bypass)->blocking_queue;
2723 /* For simplicity, just swap the BQ for the BH */
2724 P_ temp = updatee_keep;
2726 updatee_keep = updatee_bypass;
2727 updatee_bypass = temp;
2729 /* Record the swap in the kept frame (below) */
2730 prev_frame->updatee = updatee_keep;
2735 TICK_UPD_SQUEEZED();
2736 UPD_IND(updatee_bypass, updatee_keep); /* this wakes the threads up */
2738 sp = (P_)frame - 1; /* sp = stuff to slide */
2739 displacement += sizeofW(StgUpdateFrame);
2742 /* No squeeze for this frame */
2743 sp = frame_bottom - 1; /* Keep the current frame */
2745 /* Do lazy black-holing.
2747 if (is_update_frame) {
2748 StgBlockingQueue *bh = (StgBlockingQueue *)frame->updatee;
2749 if (bh->header.info != &BLACKHOLE_BQ_info &&
2750 bh->header.info != &CAF_BLACKHOLE_info) {
2751 SET_INFO(bh,&BLACKHOLE_info);
2755 /* Fix the link in the current frame (should point to the frame below) */
2756 frame->link = prev_frame;
2757 prev_was_update_frame = is_update_frame;
2760 /* Now slide all words from sp up to the next frame */
2762 if (displacement > 0) {
2763 P_ next_frame_bottom;
2765 if (next_frame != NULL)
2766 next_frame_bottom = (P_)next_frame + sizeofW(StgUpdateFrame);
2768 next_frame_bottom = tso->sp - 1;
2771 fprintf(stderr, "sliding [%p, %p] by %ld\n", sp, next_frame_bottom,
2775 while (sp >= next_frame_bottom) {
2776 sp[displacement] = *sp;
2780 (P_)prev_frame = (P_)frame + displacement;
2784 tso->sp += displacement;
2785 tso->su = prev_frame;
2788 /* -----------------------------------------------------------------------------
2791 * We have to prepare for GC - this means doing lazy black holing
2792 * here. We also take the opportunity to do stack squeezing if it's
2794 * -------------------------------------------------------------------------- */
2797 threadPaused(StgTSO *tso)
2799 if ( RtsFlags.GcFlags.squeezeUpdFrames == rtsTrue )
2800 threadSqueezeStack(tso); /* does black holing too */
2802 threadLazyBlackHole(tso);