1 /* ---------------------------------------------------------------------------
3 * (c) The GHC Team, 2000-2006
5 * Sparking support for PARALLEL_HASKELL and THREADED_RTS versions of the RTS.
7 * -------------------------------------------------------------------------*/
9 #include "PosixSource.h"
17 # if defined(PARALLEL_HASKELL)
18 # include "ParallelRts.h"
19 # include "GranSimRts.h" // for GR_...
21 # include "GranSimRts.h"
26 #if defined(THREADED_RTS) || defined(PARALLEL_HASKELL)
28 static INLINE_ME void bump_hd (StgSparkPool *p)
29 { p->hd++; if (p->hd == p->lim) p->hd = p->base; }
31 static INLINE_ME void bump_tl (StgSparkPool *p)
32 { p->tl++; if (p->tl == p->lim) p->tl = p->base; }
34 /* -----------------------------------------------------------------------------
36 * Initialising spark pools.
38 * -------------------------------------------------------------------------- */
41 initSparkPool(StgSparkPool *pool)
43 pool->base = stgMallocBytes(RtsFlags.ParFlags.maxLocalSparks
44 * sizeof(StgClosure *),
46 pool->lim = pool->base + RtsFlags.ParFlags.maxLocalSparks;
47 pool->hd = pool->base;
48 pool->tl = pool->base;
52 initSparkPools( void )
55 /* walk over the capabilities, allocating a spark pool for each one */
57 for (i = 0; i < n_capabilities; i++) {
58 initSparkPool(&capabilities[i].r.rSparks);
61 /* allocate a single spark pool */
62 initSparkPool(&MainCapability.r.rSparks);
67 freeSparkPool(StgSparkPool *pool) {
71 /* -----------------------------------------------------------------------------
73 * findSpark: find a spark on the current Capability that we can fork
76 * -------------------------------------------------------------------------- */
79 findSpark (Capability *cap)
84 pool = &(cap->r.rSparks);
85 ASSERT_SPARK_POOL_INVARIANTS(pool);
87 while (pool->hd != pool->tl) {
90 if (closure_SHOULD_SPARK(spark)) {
92 if (RtsFlags.ParFlags.ParStats.Sparks)
93 DumpRawGranEvent(CURRENT_PROC, CURRENT_PROC,
94 GR_STEALING, ((StgTSO *)NULL), spark,
95 0, 0 /* spark_queue_len(ADVISORY_POOL) */);
100 // spark pool is now empty
104 /* -----------------------------------------------------------------------------
105 * Mark all nodes pointed to by sparks in the spark queues (for GC) Does an
106 * implicit slide i.e. after marking all sparks are at the beginning of the
107 * spark pool and the spark pool only contains sparkable closures
108 * -------------------------------------------------------------------------- */
111 markSparkQueue (evac_fn evac)
113 StgClosure **sparkp, **to_sparkp;
114 nat i, n, pruned_sparks; // stats only
118 PAR_TICKY_MARK_SPARK_QUEUE_START();
122 for (i = 0; i < n_capabilities; i++) {
123 cap = &capabilities[i];
124 pool = &(cap->r.rSparks);
126 ASSERT_SPARK_POOL_INVARIANTS(pool);
128 #if defined(PARALLEL_HASKELL)
135 to_sparkp = pool->hd;
136 while (sparkp != pool->tl) {
137 ASSERT(*sparkp!=NULL);
138 ASSERT(LOOKS_LIKE_CLOSURE_PTR(((StgClosure *)*sparkp)));
139 // ToDo?: statistics gathering here (also for GUM!)
140 if (closure_SHOULD_SPARK(*sparkp)) {
142 *to_sparkp++ = *sparkp;
143 if (to_sparkp == pool->lim) {
144 to_sparkp = pool->base;
151 if (sparkp == pool->lim) {
155 pool->tl = to_sparkp;
157 PAR_TICKY_MARK_SPARK_QUEUE_END(n);
159 #if defined(PARALLEL_HASKELL)
160 debugTrace(DEBUG_sched,
161 "marked %d sparks and pruned %d sparks on [%x]",
162 n, pruned_sparks, mytid);
164 debugTrace(DEBUG_sched,
165 "marked %d sparks and pruned %d sparks",
169 debugTrace(DEBUG_sched,
170 "new spark queue len=%d; (hd=%p; tl=%p)\n",
171 sparkPoolSize(pool), pool->hd, pool->tl);
175 /* -----------------------------------------------------------------------------
177 * Turn a spark into a real thread
179 * -------------------------------------------------------------------------- */
182 createSparkThread (Capability *cap, StgClosure *p)
186 tso = createGenThread (cap, RtsFlags.GcFlags.initialStkSize, p);
187 appendToRunQueue(cap,tso);
190 /* -----------------------------------------------------------------------------
194 * -------------------------------------------------------------------------- */
199 newSpark (StgRegTable *reg, StgClosure *p)
201 StgSparkPool *pool = &(reg->rSparks);
203 /* I am not sure whether this is the right thing to do.
204 * Maybe it is better to exploit the tag information
205 * instead of throwing it away?
207 p = UNTAG_CLOSURE(p);
209 ASSERT_SPARK_POOL_INVARIANTS(pool);
211 if (closure_SHOULD_SPARK(p)) {
214 new_tl = pool->tl + 1;
215 if (new_tl == pool->lim) { new_tl = pool->base; }
216 if (new_tl != pool->hd) {
219 } else if (!closure_SHOULD_SPARK(*pool->hd)) {
220 // if the old closure is not sparkable, discard it and
221 // keep the new one. Otherwise, keep the old one.
225 #else /* DISCARD OLD */
228 if (pool->tl == pool->hd) { bump_hd(pool); }
232 ASSERT_SPARK_POOL_INVARIANTS(pool);
239 newSpark (StgRegTable *reg STG_UNUSED, StgClosure *p STG_UNUSED)
245 #endif /* PARALLEL_HASKELL || THREADED_RTS */
248 /* -----------------------------------------------------------------------------
250 * GRAN & PARALLEL_HASKELL stuff beyond here.
252 * -------------------------------------------------------------------------- */
254 #if defined(PARALLEL_HASKELL) || defined(GRAN)
256 static void slide_spark_pool( StgSparkPool *pool );
259 add_to_spark_queue( StgClosure *closure, StgSparkPool *pool )
261 if (pool->tl == pool->lim)
262 slide_spark_pool(pool);
264 if (closure_SHOULD_SPARK(closure) &&
265 pool->tl < pool->lim) {
266 *(pool->tl++) = closure;
268 #if defined(PARALLEL_HASKELL)
269 // collect parallel global statistics (currently done together with GC stats)
270 if (RtsFlags.ParFlags.ParStats.Global &&
271 RtsFlags.GcFlags.giveStats > NO_GC_STATS) {
272 // debugBelch("Creating spark for %x @ %11.2f\n", closure, usertime());
273 globalParStats.tot_sparks_created++;
278 #if defined(PARALLEL_HASKELL)
279 // collect parallel global statistics (currently done together with GC stats)
280 if (RtsFlags.ParFlags.ParStats.Global &&
281 RtsFlags.GcFlags.giveStats > NO_GC_STATS) {
282 //debugBelch("Ignoring spark for %x @ %11.2f\n", closure, usertime());
283 globalParStats.tot_sparks_ignored++;
291 slide_spark_pool( StgSparkPool *pool )
293 StgClosure **sparkp, **to_sparkp;
296 to_sparkp = pool->base;
297 while (sparkp < pool->tl) {
298 ASSERT(to_sparkp<=sparkp);
299 ASSERT(*sparkp!=NULL);
300 ASSERT(LOOKS_LIKE_GHC_INFO((*sparkp)->header.info));
302 if (closure_SHOULD_SPARK(*sparkp)) {
303 *to_sparkp++ = *sparkp++;
308 pool->hd = pool->base;
309 pool->tl = to_sparkp;
316 #if !defined(THREADED_RTS)
321 pool = &(cap->rSparks);
322 ASSERT(pool->hd <= pool->tl && pool->tl <= pool->lim);
324 ASSERT(spark != (StgClosure *)NULL);
332 Search the spark queue of the proc in event for a spark that's worth
333 turning into a thread
334 (was gimme_spark in the old RTS)
337 findLocalSpark (rtsEvent *event, rtsBool *found_res, rtsSparkQ *spark_res)
339 PEs proc = event->proc, /* proc to search for work */
340 creator = event->creator; /* proc that requested work */
343 rtsSparkQ spark_of_non_local_node = NULL,
344 spark_of_non_local_node_prev = NULL,
345 low_priority_spark = NULL,
346 low_priority_spark_prev = NULL,
347 spark = NULL, prev = NULL;
349 /* Choose a spark from the local spark queue */
350 prev = (rtsSpark*)NULL;
351 spark = pending_sparks_hds[proc];
354 // ToDo: check this code & implement local sparking !! -- HWL
355 while (!found && spark != (rtsSpark*)NULL)
357 ASSERT((prev!=(rtsSpark*)NULL || spark==pending_sparks_hds[proc]) &&
358 (prev==(rtsSpark*)NULL || prev->next==spark) &&
359 (spark->prev==prev));
361 if (!closure_SHOULD_SPARK(node))
363 IF_GRAN_DEBUG(checkSparkQ,
364 debugBelch("^^ pruning spark %p (node %p) in gimme_spark",
367 if (RtsFlags.GranFlags.GranSimStats.Sparks)
368 DumpRawGranEvent(proc, (PEs)0, SP_PRUNED,(StgTSO*)NULL,
369 spark->node, spark->name, spark_queue_len(proc));
371 ASSERT(spark != (rtsSpark*)NULL);
372 ASSERT(SparksAvail>0);
375 ASSERT(prev==(rtsSpark*)NULL || prev->next==spark);
376 spark = delete_from_sparkq (spark, proc, rtsTrue);
377 if (spark != (rtsSpark*)NULL)
381 /* -- node should eventually be sparked */
382 else if (RtsFlags.GranFlags.PreferSparksOfLocalNodes &&
383 !IS_LOCAL_TO(PROCS(node),CurrentProc))
385 barf("Local sparking not yet implemented");
387 /* Remember first low priority spark */
388 if (spark_of_non_local_node==(rtsSpark*)NULL) {
389 spark_of_non_local_node_prev = prev;
390 spark_of_non_local_node = spark;
393 if (spark->next == (rtsSpark*)NULL) {
394 /* ASSERT(spark==SparkQueueTl); just for testing */
395 prev = spark_of_non_local_node_prev;
396 spark = spark_of_non_local_node;
401 # if defined(GRAN) && defined(GRAN_CHECK)
402 /* Should never happen; just for testing
403 if (spark==pending_sparks_tl) {
404 debugBelch("ReSchedule: Last spark != SparkQueueTl\n");
405 stg_exit(EXIT_FAILURE);
410 ASSERT(SparksAvail>0);
414 else if ( RtsFlags.GranFlags.DoPrioritySparking ||
415 (spark->gran_info >= RtsFlags.GranFlags.SparkPriority2) )
417 if (RtsFlags.GranFlags.DoPrioritySparking)
418 barf("Priority sparking not yet implemented");
423 else /* only used if SparkPriority2 is defined */
425 /* ToDo: fix the code below and re-integrate it */
426 /* Remember first low priority spark */
427 if (low_priority_spark==(rtsSpark*)NULL) {
428 low_priority_spark_prev = prev;
429 low_priority_spark = spark;
432 if (spark->next == (rtsSpark*)NULL) {
433 /* ASSERT(spark==spark_queue_tl); just for testing */
434 prev = low_priority_spark_prev;
435 spark = low_priority_spark;
436 found = rtsTrue; /* take low pri spark => rc is 2 */
440 /* Should never happen; just for testing
441 if (spark==pending_sparks_tl) {
442 debugBelch("ReSchedule: Last spark != SparkQueueTl\n");
443 stg_exit(EXIT_FAILURE);
450 debugBelch("++ Ignoring spark of priority %u (SparkPriority=%u); node=%p; name=%u\n",
451 spark->gran_info, RtsFlags.GranFlags.SparkPriority,
452 spark->node, spark->name);)
455 } /* while (spark!=NULL && !found) */
462 Turn the spark into a thread.
463 In GranSim this basically means scheduling a StartThread event for the
464 node pointed to by the spark at some point in the future.
465 (was munch_spark in the old RTS)
468 activateSpark (rtsEvent *event, rtsSparkQ spark)
470 PEs proc = event->proc, /* proc to search for work */
471 creator = event->creator; /* proc that requested work */
474 rtsTime spark_arrival_time;
477 We've found a node on PE proc requested by PE creator.
478 If proc==creator we can turn the spark into a thread immediately;
479 otherwise we schedule a MoveSpark event on the requesting PE
485 /* only possible if we simulate GUM style fishing */
486 ASSERT(RtsFlags.GranFlags.Fishing);
488 /* Message packing costs for sending a Fish; qeq jabbI'ID */
489 CurrentTime[proc] += RtsFlags.GranFlags.Costs.mpacktime;
491 if (RtsFlags.GranFlags.GranSimStats.Sparks)
492 DumpRawGranEvent(proc, (PEs)0, SP_EXPORTED,
493 (StgTSO*)NULL, spark->node,
494 spark->name, spark_queue_len(proc));
496 /* time of the spark arrival on the remote PE */
497 spark_arrival_time = CurrentTime[proc] + RtsFlags.GranFlags.Costs.latency;
499 new_event(creator, proc, spark_arrival_time,
501 (StgTSO*)NULL, spark->node, spark);
503 CurrentTime[proc] += RtsFlags.GranFlags.Costs.mtidytime;
505 } else { /* proc==creator i.e. turn the spark into a thread */
507 if ( RtsFlags.GranFlags.GranSimStats.Global &&
508 spark->gran_info < RtsFlags.GranFlags.SparkPriority2 ) {
510 globalGranStats.tot_low_pri_sparks++;
512 debugBelch("++ No high priority spark available; low priority (%u) spark chosen: node=%p; name=%u\n",
514 spark->node, spark->name));
517 CurrentTime[proc] += RtsFlags.GranFlags.Costs.threadcreatetime;
522 /* ToDo: fix the GC interface and move to StartThread handling-- HWL */
523 if (GARBAGE COLLECTION IS NECESSARY) {
524 /* Some kind of backoff needed here in case there's too little heap */
525 # if defined(GRAN_CHECK) && defined(GRAN)
526 if (RtsFlags.GcFlags.giveStats)
527 fprintf(RtsFlags.GcFlags.statsFile,"***** vIS Qu' chen veQ boSwI'; spark=%p, node=%p; name=%u\n",
528 /* (found==2 ? "no hi pri spark" : "hi pri spark"), */
529 spark, node, spark->name);
531 new_event(CurrentProc, CurrentProc, CurrentTime[CurrentProc]+1,
533 (StgTSO*)NULL, (StgClosure*)NULL, (rtsSpark*)NULL);
534 barf("//// activateSpark: out of heap ; ToDo: call GarbageCollect()");
535 GarbageCollect(GetRoots, rtsFalse);
536 // HWL old: ReallyPerformThreadGC(TSO_HS+TSO_CTS_SIZE,rtsFalse);
537 // HWL old: SAVE_Hp -= TSO_HS+TSO_CTS_SIZE;
539 return; /* was: continue; */ /* to the next event, eventually */
543 if (RtsFlags.GranFlags.GranSimStats.Sparks)
544 DumpRawGranEvent(CurrentProc,(PEs)0,SP_USED,(StgTSO*)NULL,
545 spark->node, spark->name,
546 spark_queue_len(CurrentProc));
548 new_event(proc, proc, CurrentTime[proc],
550 END_TSO_QUEUE, node, spark); // (rtsSpark*)NULL);
552 procStatus[proc] = Starting;
556 /* -------------------------------------------------------------------------
557 This is the main point where handling granularity information comes into
559 ------------------------------------------------------------------------- */
561 #define MAX_RAND_PRI 100
564 Granularity info transformers.
565 Applied to the GRAN_INFO field of a spark.
567 STATIC_INLINE nat ID(nat x) { return(x); };
568 STATIC_INLINE nat INV(nat x) { return(-x); };
569 STATIC_INLINE nat IGNORE(nat x) { return (0); };
570 STATIC_INLINE nat RAND(nat x) { return ((random() % MAX_RAND_PRI) + 1); }
572 /* NB: size_info and par_info are currently unused (what a shame!) -- HWL */
574 newSpark(node,name,gran_info,size_info,par_info,local)
576 nat name, gran_info, size_info, par_info, local;
581 pri = RtsFlags.GranFlags.RandomPriorities ? RAND(gran_info) :
582 RtsFlags.GranFlags.InversePriorities ? INV(gran_info) :
583 RtsFlags.GranFlags.IgnorePriorities ? IGNORE(gran_info) :
586 if ( RtsFlags.GranFlags.SparkPriority!=0 &&
587 pri<RtsFlags.GranFlags.SparkPriority ) {
589 debugBelch(",, NewSpark: Ignoring spark of priority %u (SparkPriority=%u); node=%#x; name=%u\n",
590 pri, RtsFlags.GranFlags.SparkPriority, node, name));
591 return ((rtsSpark*)NULL);
594 newspark = (rtsSpark*) stgMallocBytes(sizeof(rtsSpark), "NewSpark");
595 newspark->prev = newspark->next = (rtsSpark*)NULL;
596 newspark->node = node;
597 newspark->name = (name==1) ? CurrentTSO->gran.sparkname : name;
598 newspark->gran_info = pri;
599 newspark->global = !local; /* Check that with parAt, parAtAbs !!*/
601 if (RtsFlags.GranFlags.GranSimStats.Global) {
602 globalGranStats.tot_sparks_created++;
603 globalGranStats.sparks_created_on_PE[CurrentProc]++;
624 disposeSparkQ(spark->next);
627 if (SparksAvail < 0) {
628 debugBelch("disposeSparkQ: SparksAvail<0 after disposing sparkq @ %p\n", &spark);
637 With PrioritySparking add_to_spark_queue performs an insert sort to keep
638 the spark queue sorted. Otherwise the spark is just added to the end of
643 add_to_spark_queue(spark)
646 rtsSpark *prev = NULL, *next = NULL;
648 rtsBool found = rtsFalse;
650 if ( spark == (rtsSpark *)NULL ) {
654 if (RtsFlags.GranFlags.DoPrioritySparking && (spark->gran_info != 0) ) {
655 /* Priority sparking is enabled i.e. spark queues must be sorted */
657 for (prev = NULL, next = pending_sparks_hd, count=0;
659 !(found = (spark->gran_info >= next->gran_info));
660 prev = next, next = next->next, count++)
663 } else { /* 'utQo' */
664 /* Priority sparking is disabled */
666 found = rtsFalse; /* to add it at the end */
671 /* next points to the first spark with a gran_info smaller than that
672 of spark; therefore, add spark before next into the spark queue */
674 if ( next == NULL ) {
675 pending_sparks_tl = spark;
680 if ( prev == NULL ) {
681 pending_sparks_hd = spark;
685 } else { /* (RtsFlags.GranFlags.DoPrioritySparking && !found) || !DoPrioritySparking */
686 /* add the spark at the end of the spark queue */
688 spark->prev = pending_sparks_tl;
689 if (pending_sparks_hd == NULL)
690 pending_sparks_hd = spark;
692 pending_sparks_tl->next = spark;
693 pending_sparks_tl = spark;
697 /* add costs for search in priority sparking */
698 if (RtsFlags.GranFlags.DoPrioritySparking) {
699 CurrentTime[CurrentProc] += count * RtsFlags.GranFlags.Costs.pri_spark_overhead;
702 IF_GRAN_DEBUG(checkSparkQ,
703 debugBelch("++ Spark stats after adding spark %p (node %p) to queue on PE %d",
704 spark, spark->node, CurrentProc);
705 print_sparkq_stats());
707 # if defined(GRAN_CHECK)
708 if (RtsFlags.GranFlags.Debug.checkSparkQ) {
709 for (prev = NULL, next = pending_sparks_hd;
711 prev = next, next = next->next)
713 if ( (prev!=NULL) && (prev!=pending_sparks_tl) )
714 debugBelch("SparkQ inconsistency after adding spark %p: (PE %u) pending_sparks_tl (%p) not end of queue (%p)\n",
716 pending_sparks_tl, prev);
720 # if defined(GRAN_CHECK)
721 /* Check if the sparkq is still sorted. Just for testing, really! */
722 if ( RtsFlags.GranFlags.Debug.checkSparkQ &&
723 RtsFlags.GranFlags.Debug.pri ) {
724 rtsBool sorted = rtsTrue;
725 rtsSpark *prev, *next;
727 if (pending_sparks_hd == NULL ||
728 pending_sparks_hd->next == NULL ) {
729 /* just 1 elem => ok */
731 for (prev = pending_sparks_hd,
732 next = pending_sparks_hd->next;
734 prev = next, next = next->next) {
736 (prev->gran_info >= next->gran_info);
740 debugBelch("ghuH: SPARKQ on PE %d is not sorted:\n",
742 print_sparkq(CurrentProc);
749 spark_queue_len(proc)
752 rtsSpark *prev, *spark; /* prev only for testing !! */
755 for (len = 0, prev = NULL, spark = pending_sparks_hds[proc];
757 len++, prev = spark, spark = spark->next)
760 # if defined(GRAN_CHECK)
761 if ( RtsFlags.GranFlags.Debug.checkSparkQ )
762 if ( (prev!=NULL) && (prev!=pending_sparks_tls[proc]) )
763 debugBelch("ERROR in spark_queue_len: (PE %u) pending_sparks_tl (%p) not end of queue (%p)\n",
764 proc, pending_sparks_tls[proc], prev);
771 Take spark out of the spark queue on PE p and nuke the spark. Adjusts
772 hd and tl pointers of the spark queue. Returns a pointer to the next
776 delete_from_sparkq (spark, p, dispose_too) /* unlink and dispose spark */
784 barf("delete_from_sparkq: trying to delete NULL spark\n");
786 # if defined(GRAN_CHECK)
787 if ( RtsFlags.GranFlags.Debug.checkSparkQ ) {
788 debugBelch("## |%p:%p| (%p)<-spark=%p->(%p) <-(%p)\n",
789 pending_sparks_hd, pending_sparks_tl,
790 spark->prev, spark, spark->next,
791 (spark->next==NULL ? 0 : spark->next->prev));
795 if (spark->prev==NULL) {
796 /* spark is first spark of queue => adjust hd pointer */
797 ASSERT(pending_sparks_hds[p]==spark);
798 pending_sparks_hds[p] = spark->next;
800 spark->prev->next = spark->next;
802 if (spark->next==NULL) {
803 ASSERT(pending_sparks_tls[p]==spark);
804 /* spark is first spark of queue => adjust tl pointer */
805 pending_sparks_tls[p] = spark->prev;
807 spark->next->prev = spark->prev;
809 new_spark = spark->next;
811 # if defined(GRAN_CHECK)
812 if ( RtsFlags.GranFlags.Debug.checkSparkQ ) {
813 debugBelch("## |%p:%p| (%p)<-spark=%p->(%p) <-(%p); spark=%p will be deleted NOW \n",
814 pending_sparks_hd, pending_sparks_tl,
815 spark->prev, spark, spark->next,
816 (spark->next==NULL ? 0 : spark->next->prev), spark);
826 /* Mark all nodes pointed to by sparks in the spark queues (for GC) */
830 StgClosure *MarkRoot(StgClosure *root); // prototype
834 for (p=0; p<RtsFlags.GranFlags.proc; p++)
835 for (sp=pending_sparks_hds[p]; sp!=NULL; sp=sp->next) {
836 ASSERT(sp->node!=NULL);
837 ASSERT(LOOKS_LIKE_GHC_INFO(sp->node->header.info));
838 // ToDo?: statistics gathering here (also for GUM!)
839 sp->node = (StgClosure *)MarkRoot(sp->node);
843 debugBelch("markSparkQueue: spark statistics at start of GC:");
844 print_sparkq_stats());
854 debugBelch("Spark: NIL\n");
858 ((spark->node==NULL) ? "______" : "%#6lx"),
859 stgCast(StgPtr,spark->node));
861 debugBelch("Spark: Node %8s, Name %#6x, Global %5s, Creator %5x, Prev %6p, Next %6p\n",
863 ((spark->global)==rtsTrue?"True":"False"), spark->creator,
864 spark->prev, spark->next);
873 rtsSpark *x = pending_sparks_hds[proc];
875 debugBelch("Spark Queue of PE %d with root at %p:\n", proc, x);
876 for (; x!=(rtsSpark*)NULL; x=x->next) {
882 Print a statistics of all spark queues.
885 print_sparkq_stats(void)
889 debugBelch("SparkQs: [");
890 for (p=0; p<RtsFlags.GranFlags.proc; p++)
891 debugBelch(", PE %d: %d", p, spark_queue_len(p));