X-Git-Url: http://git.megacz.com/?p=ghc-hetmet.git;a=blobdiff_plain;f=includes%2FStorage.h;h=5d3e7733cf033d7cc2cd28e1fd77eb0a702bea30;hp=1886e0921ec7c7f662eb4d2a9e327539f76fd97f;hb=5d379cbe65e406d5c3a848fe7fcd090cafbfeb78;hpb=e3c55aebd4f9ce7a5b4390d4726612865fd207f2 diff --git a/includes/Storage.h b/includes/Storage.h index 1886e09..5d3e773 100644 --- a/includes/Storage.h +++ b/includes/Storage.h @@ -11,6 +11,7 @@ #include #include "OSThreads.h" +#include "SMP.h" /* ----------------------------------------------------------------------------- * Generational GC @@ -52,49 +53,72 @@ * ------------------------------------------------------------------------- */ typedef struct step_ { - unsigned int no; /* step number */ - bdescr * blocks; /* blocks in this step */ - unsigned int n_blocks; /* number of blocks */ - struct step_ * to; /* destination step for live objects */ - struct generation_ * gen; /* generation this step belongs to */ - unsigned int gen_no; /* generation number (cached) */ - bdescr * large_objects; /* large objects (doubly linked) */ - unsigned int n_large_blocks; /* no. of blocks used by large objs */ - int is_compacted; /* compact this step? (old gen only) */ - - /* During GC, if we are collecting this step, blocks and n_blocks - * are copied into the following two fields. After GC, these blocks - * are freed. */ - bdescr * old_blocks; /* bdescr of first from-space block */ - unsigned int n_old_blocks; /* number of blocks in from-space */ - - /* temporary use during GC: */ - StgPtr hp; /* next free locn in to-space */ - StgPtr hpLim; /* end of current to-space block */ - bdescr * hp_bd; /* bdescr of current to-space block */ - StgPtr scavd_hp; /* ... same as above, but already */ - StgPtr scavd_hpLim; /* scavenged. */ - bdescr * scan_bd; /* block currently being scanned */ - StgPtr scan; /* scan pointer in current block */ - bdescr * new_large_objects; /* large objects collected so far */ - bdescr * scavenged_large_objects; /* live large objs after GC (d-link) */ - unsigned int n_scavenged_large_blocks;/* size of above */ - bdescr * bitmap; /* bitmap for compacting collection */ + unsigned int no; // step number in this generation + unsigned int abs_no; // absolute step number + + struct generation_ * gen; // generation this step belongs to + unsigned int gen_no; // generation number (cached) + + bdescr * blocks; // blocks in this step + unsigned int n_blocks; // number of blocks + unsigned int n_words; // number of words + + struct step_ * to; // destination step for live objects + + bdescr * large_objects; // large objects (doubly linked) + unsigned int n_large_blocks; // no. of blocks used by large objs + + StgTSO * threads; // threads in this step + // linked via global_link + + // ------------------------------------ + // Fields below are used during GC only + + // During GC, if we are collecting this step, blocks and n_blocks + // are copied into the following two fields. After GC, these blocks + // are freed. + +#if defined(THREADED_RTS) + char pad[128]; // make sure the following is + // on a separate cache line. + SpinLock sync_large_objects; // lock for large_objects + // and scavenged_large_objects +#endif + + int mark; // mark (not copy)? (old gen only) + int compact; // compact (not sweep)? (old gen only) + + bdescr * old_blocks; // bdescr of first from-space block + unsigned int n_old_blocks; // number of blocks in from-space + unsigned int live_estimate; // for sweeping: estimate of live data + + bdescr * part_blocks; // partially-full scanned blocks + unsigned int n_part_blocks; // count of above + + bdescr * scavenged_large_objects; // live large objs after GC (d-link) + unsigned int n_scavenged_large_blocks; // size (not count) of above + + bdescr * bitmap; // bitmap for compacting collection + + StgTSO * old_threads; + } step; + typedef struct generation_ { - unsigned int no; /* generation number */ - step * steps; /* steps */ - unsigned int n_steps; /* number of steps */ - unsigned int max_blocks; /* max blocks in step 0 */ - bdescr *mut_list; /* mut objects in this gen (not G0)*/ - - /* temporary use during GC: */ - bdescr *saved_mut_list; - - /* stats information */ - unsigned int collections; - unsigned int failed_promotions; + unsigned int no; // generation number + step * steps; // steps + unsigned int n_steps; // number of steps + unsigned int max_blocks; // max blocks in step 0 + bdescr *mut_list; // mut objects in this gen (not G0) + + // stats information + unsigned int collections; + unsigned int par_collections; + unsigned int failed_promotions; + + // temporary use during GC: + bdescr *saved_mut_list; } generation; extern generation * RTS_VAR(generations); @@ -102,6 +126,8 @@ extern generation * RTS_VAR(generations); extern generation * RTS_VAR(g0); extern step * RTS_VAR(g0s0); extern generation * RTS_VAR(oldest_gen); +extern step * RTS_VAR(all_steps); +extern nat RTS_VAR(total_steps); /* ----------------------------------------------------------------------------- Initialisation / De-initialisation @@ -114,10 +140,20 @@ extern void freeStorage(void); /* ----------------------------------------------------------------------------- Generic allocation - StgPtr allocate(nat n) Allocates a chunk of contiguous store - n words long, returning a pointer to - the first word. Always succeeds. + StgPtr allocateInGen(generation *g, nat n) + Allocates a chunk of contiguous store + n words long in generation g, + returning a pointer to the first word. + Always succeeds. + StgPtr allocate(nat n) Equaivalent to allocateInGen(g0) + + StgPtr allocateLocal(Capability *cap, nat n) + Allocates memory from the nursery in + the current Capability. This can be + done without taking a global lock, + unlike allocate(). + StgPtr allocatePinned(nat n) Allocates a chunk of contiguous store n words long, which is at a fixed address (won't be moved by GC). @@ -137,26 +173,22 @@ extern void freeStorage(void); rtsBool doYouWantToGC(void) Returns True if the storage manager is ready to perform a GC, False otherwise. - lnat allocated_bytes(void) Returns the number of bytes allocated + lnat allocatedBytes(void) Returns the number of bytes allocated via allocate() since the last GC. Used in the reporting of statistics. - THREADED_RTS: allocate and doYouWantToGC can be used from STG code, they are - surrounded by a mutex. -------------------------------------------------------------------------- */ -extern StgPtr allocate ( nat n ); -extern StgPtr allocateLocal ( Capability *cap, nat n ); -extern StgPtr allocatePinned ( nat n ); -extern lnat allocated_bytes ( void ); +extern StgPtr allocate ( lnat n ); +extern StgPtr allocateInGen ( generation *g, lnat n ); +extern StgPtr allocateLocal ( Capability *cap, lnat n ); +extern StgPtr allocatePinned ( lnat n ); +extern lnat allocatedBytes ( void ); extern bdescr * RTS_VAR(small_alloc_list); extern bdescr * RTS_VAR(large_alloc_list); extern bdescr * RTS_VAR(pinned_object_block); -extern StgPtr RTS_VAR(alloc_Hp); -extern StgPtr RTS_VAR(alloc_HpLim); - extern nat RTS_VAR(alloc_blocks); extern nat RTS_VAR(alloc_blocks_lim); @@ -167,9 +199,12 @@ doYouWantToGC( void ) } /* memory allocator for executable memory */ -extern void *allocateExec (nat bytes); +extern void* allocateExec(unsigned int len, void **exec_addr); extern void freeExec (void *p); +/* for splitting blocks groups in two */ +extern bdescr * splitLargeBlock (bdescr *bd, nat blocks); + /* ----------------------------------------------------------------------------- Performing Garbage Collection @@ -177,11 +212,10 @@ extern void freeExec (void *p); 'get_roots' is called to find all the roots that the system knows about. - StgClosure Called by get_roots on each root. - MarkRoot(StgClosure *p) Returns the new location of the root. + -------------------------------------------------------------------------- */ -extern void GarbageCollect(void (*get_roots)(evac_fn),rtsBool force_major_gc); +extern void GarbageCollect(rtsBool force_major_gc, nat gc_type, Capability *cap); /* ----------------------------------------------------------------------------- Generational garbage collection support @@ -205,7 +239,6 @@ extern void GarbageCollect(void (*get_roots)(evac_fn),rtsBool force_major_gc); */ #if defined(THREADED_RTS) extern Mutex sm_mutex; -extern Mutex atomic_modify_mutvar_mutex; #endif #if defined(THREADED_RTS) @@ -218,28 +251,30 @@ extern Mutex atomic_modify_mutvar_mutex; #define ASSERT_SM_LOCK() #endif +#if !IN_STG_CODE + INLINE_HEADER void -recordMutableGen(StgClosure *p, generation *gen) +recordMutableGen(StgClosure *p, nat gen_no) { bdescr *bd; - bd = gen->mut_list; + bd = generations[gen_no].mut_list; if (bd->free >= bd->start + BLOCK_SIZE_W) { bdescr *new_bd; new_bd = allocBlock(); new_bd->link = bd; bd = new_bd; - gen->mut_list = bd; + generations[gen_no].mut_list = bd; } *bd->free++ = (StgWord)p; } INLINE_HEADER void -recordMutableGenLock(StgClosure *p, generation *gen) +recordMutableGenLock(StgClosure *p, nat gen_no) { ACQUIRE_SM_LOCK; - recordMutableGen(p,gen); + recordMutableGen(p,gen_no); RELEASE_SM_LOCK; } @@ -249,7 +284,7 @@ recordMutable(StgClosure *p) bdescr *bd; ASSERT(closure_MUTABLE(p)); bd = Bdescr((P_)p); - if (bd->gen_no > 0) recordMutableGen(p, &RTS_DEREF(generations)[bd->gen_no]); + if (bd->gen_no > 0) recordMutableGen(p, bd->gen_no); } INLINE_HEADER void @@ -260,6 +295,8 @@ recordMutableLock(StgClosure *p) RELEASE_SM_LOCK; } +#endif // !IN_STG_CODE + /* ----------------------------------------------------------------------------- The CAF table - used to let us revert CAFs in GHCi -------------------------------------------------------------------------- */ @@ -294,12 +331,8 @@ void dirty_MUT_VAR(StgRegTable *reg, StgClosure *p); make sense... -------------------------------------------------------------------------- */ -#define LOOKS_LIKE_INFO_PTR(p) \ - (p && ((StgInfoTable *)(INFO_PTR_TO_STRUCT(p)))->type != INVALID_OBJECT && \ - ((StgInfoTable *)(INFO_PTR_TO_STRUCT(p)))->type < N_CLOSURE_TYPES) - -#define LOOKS_LIKE_CLOSURE_PTR(p) \ - (LOOKS_LIKE_INFO_PTR(((StgClosure *)(p))->header.info)) +INLINE_HEADER rtsBool LOOKS_LIKE_INFO_PTR (StgWord p); +INLINE_HEADER rtsBool LOOKS_LIKE_CLOSURE_PTR (void *p); // XXX StgClosure* /* ----------------------------------------------------------------------------- Macros for calculating how big a closure will be (used during allocation) @@ -358,7 +391,7 @@ INLINE_HEADER StgWord tso_sizeW ( StgTSO *tso ) INLINE_HEADER StgWord bco_sizeW ( StgBCO *bco ) { return bco->size; } -STATIC_INLINE nat +INLINE_HEADER nat closure_sizeW_ (StgClosure *p, StgInfoTable *info) { switch (info->type) { @@ -388,6 +421,7 @@ closure_sizeW_ (StgClosure *p, StgInfoTable *info) case AP_STACK: return ap_stack_sizeW((StgAP_STACK *)p); case AP: + return ap_sizeW((StgAP *)p); case PAP: return pap_sizeW((StgPAP *)p); case IND: @@ -406,21 +440,25 @@ closure_sizeW_ (StgClosure *p, StgInfoTable *info) return tso_sizeW((StgTSO *)p); case BCO: return bco_sizeW((StgBCO *)p); - case TVAR_WAIT_QUEUE: - return sizeofW(StgTVarWaitQueue); + case TVAR_WATCH_QUEUE: + return sizeofW(StgTVarWatchQueue); case TVAR: return sizeofW(StgTVar); case TREC_CHUNK: return sizeofW(StgTRecChunk); case TREC_HEADER: return sizeofW(StgTRecHeader); + case ATOMIC_INVARIANT: + return sizeofW(StgAtomicInvariant); + case INVARIANT_CHECK_QUEUE: + return sizeofW(StgInvariantCheckQueue); default: return sizeW_fromITBL(info); } } // The definitive way to find the size, in words, of a heap-allocated closure -STATIC_INLINE nat +INLINE_HEADER nat closure_sizeW (StgClosure *p) { return closure_sizeW_(p, get_itbl(p)); @@ -449,7 +487,6 @@ INLINE_HEADER StgWord stack_frame_sizeW( StgClosure *frame ) return sizeofW(StgRetFun) + ((StgRetFun *)frame)->size; case RET_BIG: - case RET_VEC_BIG: return 1 + GET_LARGE_BITMAP(&info->i)->size; case RET_BCO: @@ -468,16 +505,19 @@ extern void allocNurseries ( void ); extern void resetNurseries ( void ); extern void resizeNurseries ( nat blocks ); extern void resizeNurseriesFixed ( nat blocks ); -extern void tidyAllocateLists ( void ); extern lnat countNurseryBlocks ( void ); + /* ----------------------------------------------------------------------------- Functions from GC.c -------------------------------------------------------------------------- */ +typedef void (*evac_fn)(void *user, StgClosure **root); + extern void threadPaused ( Capability *cap, StgTSO * ); extern StgClosure * isAlive ( StgClosure *p ); -extern void markCAFs ( evac_fn evac ); +extern void markCAFs ( evac_fn evac, void *user ); +extern void GetRoots ( evac_fn evac, void *user ); /* ----------------------------------------------------------------------------- Stats 'n' DEBUG stuff @@ -486,11 +526,13 @@ extern void markCAFs ( evac_fn evac ); extern ullong RTS_VAR(total_allocated); extern lnat calcAllocated ( void ); -extern lnat calcLive ( void ); +extern lnat calcLiveBlocks ( void ); +extern lnat calcLiveWords ( void ); +extern lnat countOccupied ( bdescr *bd ); extern lnat calcNeeded ( void ); #if defined(DEBUG) -extern void memInventory(void); +extern void memInventory(rtsBool show); extern void checkSanity(void); extern nat countBlocks(bdescr *); extern void checkNurserySanity( step *stp ); @@ -512,11 +554,30 @@ extern void newDynCAF(StgClosure *); extern void move_TSO(StgTSO *src, StgTSO *dest); extern StgTSO *relocate_stack(StgTSO *dest, ptrdiff_t diff); -extern StgClosure * RTS_VAR(scavenged_static_objects); extern StgWeak * RTS_VAR(old_weak_ptr_list); extern StgWeak * RTS_VAR(weak_ptr_list); extern StgClosure * RTS_VAR(caf_list); extern StgClosure * RTS_VAR(revertible_caf_list); extern StgTSO * RTS_VAR(resurrected_threads); +#define IS_FORWARDING_PTR(p) ((((StgWord)p) & 1) != 0) +#define MK_FORWARDING_PTR(p) (((StgWord)p) | 1) +#define UN_FORWARDING_PTR(p) (((StgWord)p) - 1) + +INLINE_HEADER rtsBool LOOKS_LIKE_INFO_PTR_NOT_NULL (StgWord p) +{ + StgInfoTable *info = INFO_PTR_TO_STRUCT(p); + return info->type != INVALID_OBJECT && info->type < N_CLOSURE_TYPES; +} + +INLINE_HEADER rtsBool LOOKS_LIKE_INFO_PTR (StgWord p) +{ + return p && (IS_FORWARDING_PTR(p) || LOOKS_LIKE_INFO_PTR_NOT_NULL(p)); +} + +INLINE_HEADER rtsBool LOOKS_LIKE_CLOSURE_PTR (void *p) +{ + return LOOKS_LIKE_INFO_PTR((StgWord)(UNTAG_CLOSURE((StgClosure *)(p)))->header.info); +} + #endif /* STORAGE_H */