[project @ 2001-01-24 15:46:19 by simonmar]
[ghc-hetmet.git] / ghc / rts / Schedule.c
index 443c7d5..63abd17 100644 (file)
@@ -1,5 +1,5 @@
 /* ---------------------------------------------------------------------------
- * $Id: Schedule.c,v 1.54 2000/03/16 17:33:04 simonmar Exp $
+ * $Id: Schedule.c,v 1.87 2001/01/24 15:46:19 simonmar Exp $
  *
  * (c) The GHC Team, 1998-2000
  *
@@ -37,7 +37,6 @@
 //@menu
 //* Includes::                 
 //* Variables and Data structures::  
-//* Prototypes::               
 //* Main scheduling loop::     
 //* Suspend and Resume::       
 //* Run queue code::           
 #include "Schedule.h"
 #include "StgMiscClosures.h"
 #include "Storage.h"
-#include "Evaluator.h"
+#include "Interpreter.h"
 #include "Exception.h"
 #include "Printer.h"
 #include "Main.h"
 #include "Signals.h"
-#include "Profiling.h"
 #include "Sanity.h"
 #include "Stats.h"
-#include "Sparks.h"
+#include "Itimer.h"
 #include "Prelude.h"
 #if defined(GRAN) || defined(PAR)
 # include "GranSimRts.h"
@@ -82,6 +80,7 @@
 # include "FetchMe.h"
 # include "HLC.h"
 #endif
+#include "Sparks.h"
 
 #include <stdarg.h>
 
@@ -136,12 +135,18 @@ StgTSO* ActiveTSO = NULL; /* for assigning system costs; GranSim-Light only */
 StgTSO *run_queue_hds[MAX_PROC], *run_queue_tls[MAX_PROC];
 StgTSO *blocked_queue_hds[MAX_PROC], *blocked_queue_tls[MAX_PROC];
 StgTSO *ccalling_threadss[MAX_PROC];
-StgTSO *all_threadss[MAX_PROC];
+/* We use the same global list of threads (all_threads) in GranSim as in
+   the std RTS (i.e. we are cheating). However, we don't use this list in
+   the GranSim specific code at the moment (so we are only potentially
+   cheating).  */
 
 #else /* !GRAN */
 
 StgTSO *run_queue_hd, *run_queue_tl;
 StgTSO *blocked_queue_hd, *blocked_queue_tl;
+StgTSO *sleeping_queue;                /* perhaps replace with a hash table? */
+
+#endif
 
 /* Linked list of all threads.
  * Used for detecting garbage collected threads.
@@ -154,7 +159,6 @@ static StgTSO *suspended_ccalling_threads;
 
 static void GetRoots(void);
 static StgTSO *threadStackOverflow(StgTSO *tso);
-#endif
 
 /* KH: The following two flags are shared memory locations.  There is no need
        to lock them, since they are only unset at the end of a scheduler
@@ -207,11 +211,15 @@ Capability MainRegTable;       /* for non-SMP, we have one global capability */
 #endif
 
 #if defined(GRAN)
-StgTSO      *CurrentTSOs[MAX_PROC];
-#else
-StgTSO      *CurrentTSO;
+StgTSO *CurrentTSO;
 #endif
 
+/*  This is used in `TSO.h' and gcc 2.96 insists that this variable actually 
+ *  exists - earlier gccs apparently didn't.
+ *  -= chak
+ */
+StgTSO dummy_tso;
+
 rtsBool ready_to_gc;
 
 /* All our current task ids, saved in case we need to kill them later.
@@ -225,7 +233,13 @@ void            addToBlockedQueue ( StgTSO *tso );
 
 static void     schedule          ( void );
        void     interruptStgRts   ( void );
+#if defined(GRAN)
+static StgTSO * createThread_     ( nat size, rtsBool have_lock, StgInt pri );
+#else
 static StgTSO * createThread_     ( nat size, rtsBool have_lock );
+#endif
+
+static void     detectBlackHoles  ( void );
 
 #ifdef DEBUG
 static void sched_belch(char *s, ...);
@@ -249,16 +263,30 @@ StgTSO *LastTSO;
 rtsTime TimeOfLastYield;
 #endif
 
+#if DEBUG
+char *whatNext_strs[] = {
+  "ThreadEnterGHC",
+  "ThreadRunGHC",
+  "ThreadEnterInterp",
+  "ThreadKilled",
+  "ThreadComplete"
+};
+
+char *threadReturnCode_strs[] = {
+  "HeapOverflow",                      /* might also be StackOverflow */
+  "StackOverflow",
+  "ThreadYielding",
+  "ThreadBlocked",
+  "ThreadFinished"
+};
+#endif
+
 /*
  * The thread state for the main thread.
 // ToDo: check whether not needed any more
 StgTSO   *MainTSO;
  */
 
-
-//@node Prototypes, Main scheduling loop, Variables and Data structures, Main scheduling code
-//@subsection Prototypes
-
 //@node Main scheduling loop, Suspend and Resume, Prototypes, Main scheduling code
 //@subsection Main scheduling loop
 
@@ -281,6 +309,21 @@ StgTSO   *MainTSO;
       * waiting for work, or
       * waiting for a GC to complete.
 
+   GRAN version:
+     In a GranSim setup this loop iterates over the global event queue.
+     This revolves around the global event queue, which determines what 
+     to do next. Therefore, it's more complicated than either the 
+     concurrent or the parallel (GUM) setup.
+
+   GUM version:
+     GUM iterates over incoming messages.
+     It starts with nothing to do (thus CurrentTSO == END_TSO_QUEUE),
+     and sends out a fish whenever it has nothing to do; in-between
+     doing the actual reductions (shared code below) it processes the
+     incoming messages and deals with delayed operations 
+     (see PendingFetches).
+     This is not the ugliest code you could imagine, but it's bloody close.
+
    ------------------------------------------------------------------------ */
 //@cindex schedule
 static void
@@ -292,6 +335,7 @@ schedule( void )
 #if defined(GRAN)
   rtsEvent *event;
 #elif defined(PAR)
+  StgSparkPool *pool;
   rtsSpark spark;
   StgTSO *tso;
   GlobalTaskId pe;
@@ -301,15 +345,37 @@ schedule( void )
   ACQUIRE_LOCK(&sched_mutex);
 
 #if defined(GRAN)
-# error ToDo: implement GranSim scheduler
+
+  /* set up first event to get things going */
+  /* ToDo: assign costs for system setup and init MainTSO ! */
+  new_event(CurrentProc, CurrentProc, CurrentTime[CurrentProc],
+           ContinueThread, 
+           CurrentTSO, (StgClosure*)NULL, (rtsSpark*)NULL);
+
+  IF_DEBUG(gran,
+          fprintf(stderr, "GRAN: Init CurrentTSO (in schedule) = %p\n", CurrentTSO);
+          G_TSO(CurrentTSO, 5));
+
+  if (RtsFlags.GranFlags.Light) {
+    /* Save current time; GranSim Light only */
+    CurrentTSO->gran.clock = CurrentTime[CurrentProc];
+  }      
+
+  event = get_next_event();
+
+  while (event!=(rtsEvent*)NULL) {
+    /* Choose the processor with the next event */
+    CurrentProc = event->proc;
+    CurrentTSO = event->tso;
+
 #elif defined(PAR)
+
   while (!GlobalStopPending) {          /* GlobalStopPending set in par_exit */
 
-    if (PendingFetches != END_BF_QUEUE) {
-        processFetches();
-    }
 #else
+
   while (1) {
+
 #endif
 
     IF_DEBUG(scheduler, printAllThreads());
@@ -320,14 +386,7 @@ schedule( void )
      */
     if (interrupted) {
       IF_DEBUG(scheduler, sched_belch("interrupted"));
-      for (t = run_queue_hd; t != END_TSO_QUEUE; t = t->link) {
-       deleteThread(t);
-      }
-      for (t = blocked_queue_hd; t != END_TSO_QUEUE; t = t->link) {
-       deleteThread(t);
-      }
-      run_queue_hd = run_queue_tl = END_TSO_QUEUE;
-      blocked_queue_hd = blocked_queue_tl = END_TSO_QUEUE;
+      deleteAllThreads();
       interrupted = rtsFalse;
       was_interrupted = rtsTrue;
     }
@@ -342,7 +401,7 @@ schedule( void )
       StgMainThread *m, **prev;
       prev = &main_threads;
       for (m = main_threads; m != NULL; m = m->link) {
-       switch (m->tso->whatNext) {
+       switch (m->tso->what_next) {
        case ThreadComplete:
          if (m->ret) {
            *(m->ret) = (StgClosure *)m->tso->sp[0];
@@ -365,15 +424,20 @@ schedule( void )
        }
       }
     }
+
 #else
+# if defined(PAR)
+    /* in GUM do this only on the Main PE */
+    if (IAmMainThread)
+# endif
     /* If our main thread has finished or been killed, return.
      */
     {
       StgMainThread *m = main_threads;
-      if (m->tso->whatNext == ThreadComplete
-         || m->tso->whatNext == ThreadKilled) {
+      if (m->tso->what_next == ThreadComplete
+         || m->tso->what_next == ThreadKilled) {
        main_threads = main_threads->link;
-       if (m->tso->whatNext == ThreadComplete) {
+       if (m->tso->what_next == ThreadComplete) {
          /* we finished successfully, fill in the return value */
          if (m->ret) { *(m->ret) = (StgClosure *)m->tso->sp[0]; };
          m->stat = Success;
@@ -442,7 +506,7 @@ schedule( void )
      * ToDo: what if another client comes along & requests another
      * main thread?
      */
-    if (blocked_queue_hd != END_TSO_QUEUE) {
+    if (blocked_queue_hd != END_TSO_QUEUE || sleeping_queue != END_TSO_QUEUE) {
       awaitEvent(
           (run_queue_hd == END_TSO_QUEUE)
 #ifdef SMP
@@ -450,40 +514,59 @@ schedule( void )
 #endif
        );
     }
-    
+    /* we can be interrupted while waiting for I/O... */
+    if (interrupted) continue;
+
     /* check for signals each time around the scheduler */
-#ifndef __MINGW32__
+#ifndef mingw32_TARGET_OS
     if (signals_pending()) {
       start_signal_handlers();
     }
 #endif
 
-    /* Detect deadlock: when we have no threads to run, there are
-     * no threads waiting on I/O or sleeping, and all the other
-     * tasks are waiting for work, we must have a deadlock.  Inform
-     * all the main threads.
+    /* 
+     * Detect deadlock: when we have no threads to run, there are no
+     * threads waiting on I/O or sleeping, and all the other tasks are
+     * waiting for work, we must have a deadlock of some description.
+     *
+     * We first try to find threads blocked on themselves (ie. black
+     * holes), and generate NonTermination exceptions where necessary.
+     *
+     * If no threads are black holed, we have a deadlock situation, so
+     * inform all the main threads.
      */
 #ifdef SMP
     if (blocked_queue_hd == END_TSO_QUEUE
        && run_queue_hd == END_TSO_QUEUE
-       && (n_free_capabilities == RtsFlags.ParFlags.nNodes)
-       ) {
-      StgMainThread *m;
-      for (m = main_threads; m != NULL; m = m->link) {
-         m->ret = NULL;
-         m->stat = Deadlock;
-         pthread_cond_broadcast(&m->wakeup);
-      }
-      main_threads = NULL;
+       && sleeping_queue == END_TSO_QUEUE
+       && (n_free_capabilities == RtsFlags.ParFlags.nNodes))
+    {
+       IF_DEBUG(scheduler, sched_belch("deadlocked, checking for black holes..."));
+       detectBlackHoles();
+       if (run_queue_hd == END_TSO_QUEUE) {
+           StgMainThread *m;
+           for (m = main_threads; m != NULL; m = m->link) {
+               m->ret = NULL;
+               m->stat = Deadlock;
+               pthread_cond_broadcast(&m->wakeup);
+           }
+           main_threads = NULL;
+       }
     }
 #else /* ! SMP */
     if (blocked_queue_hd == END_TSO_QUEUE
-       && run_queue_hd == END_TSO_QUEUE) {
-      StgMainThread *m = main_threads;
-      m->ret = NULL;
-      m->stat = Deadlock;
-      main_threads = m->link;
-      return;
+       && run_queue_hd == END_TSO_QUEUE
+       && sleeping_queue == END_TSO_QUEUE)
+    {
+       IF_DEBUG(scheduler, sched_belch("deadlocked, checking for black holes..."));
+       detectBlackHoles();
+       if (run_queue_hd == END_TSO_QUEUE) {
+           StgMainThread *m = main_threads;
+           m->ret = NULL;
+           m->stat = Deadlock;
+           main_threads = m->link;
+           return;
+       }
     }
 #endif
 
@@ -507,15 +590,156 @@ schedule( void )
 #endif
 
 #if defined(GRAN)
-# error ToDo: implement GranSim scheduler
+
+    if (RtsFlags.GranFlags.Light)
+      GranSimLight_enter_system(event, &ActiveTSO); // adjust ActiveTSO etc
+
+    /* adjust time based on time-stamp */
+    if (event->time > CurrentTime[CurrentProc] &&
+        event->evttype != ContinueThread)
+      CurrentTime[CurrentProc] = event->time;
+    
+    /* Deal with the idle PEs (may issue FindWork or MoveSpark events) */
+    if (!RtsFlags.GranFlags.Light)
+      handleIdlePEs();
+
+    IF_DEBUG(gran, fprintf(stderr, "GRAN: switch by event-type\n"))
+
+    /* main event dispatcher in GranSim */
+    switch (event->evttype) {
+      /* Should just be continuing execution */
+    case ContinueThread:
+      IF_DEBUG(gran, fprintf(stderr, "GRAN: doing ContinueThread\n"));
+      /* ToDo: check assertion
+      ASSERT(run_queue_hd != (StgTSO*)NULL &&
+            run_queue_hd != END_TSO_QUEUE);
+      */
+      /* Ignore ContinueThreads for fetching threads (if synchr comm) */
+      if (!RtsFlags.GranFlags.DoAsyncFetch &&
+         procStatus[CurrentProc]==Fetching) {
+       belch("ghuH: Spurious ContinueThread while Fetching ignored; TSO %d (%p) [PE %d]",
+             CurrentTSO->id, CurrentTSO, CurrentProc);
+       goto next_thread;
+      }        
+      /* Ignore ContinueThreads for completed threads */
+      if (CurrentTSO->what_next == ThreadComplete) {
+       belch("ghuH: found a ContinueThread event for completed thread %d (%p) [PE %d] (ignoring ContinueThread)", 
+             CurrentTSO->id, CurrentTSO, CurrentProc);
+       goto next_thread;
+      }        
+      /* Ignore ContinueThreads for threads that are being migrated */
+      if (PROCS(CurrentTSO)==Nowhere) { 
+       belch("ghuH: trying to run the migrating TSO %d (%p) [PE %d] (ignoring ContinueThread)",
+             CurrentTSO->id, CurrentTSO, CurrentProc);
+       goto next_thread;
+      }
+      /* The thread should be at the beginning of the run queue */
+      if (CurrentTSO!=run_queue_hds[CurrentProc]) { 
+       belch("ghuH: TSO %d (%p) [PE %d] is not at the start of the run_queue when doing a ContinueThread",
+             CurrentTSO->id, CurrentTSO, CurrentProc);
+       break; // run the thread anyway
+      }
+      /*
+      new_event(proc, proc, CurrentTime[proc],
+               FindWork,
+               (StgTSO*)NULL, (StgClosure*)NULL, (rtsSpark*)NULL);
+      goto next_thread; 
+      */ /* Catches superfluous CONTINUEs -- should be unnecessary */
+      break; // now actually run the thread; DaH Qu'vam yImuHbej 
+
+    case FetchNode:
+      do_the_fetchnode(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case GlobalBlock:
+      do_the_globalblock(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case FetchReply:
+      do_the_fetchreply(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case UnblockThread:   /* Move from the blocked queue to the tail of */
+      do_the_unblock(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case ResumeThread:  /* Move from the blocked queue to the tail of */
+      /* the runnable queue ( i.e. Qu' SImqa'lu') */ 
+      event->tso->gran.blocktime += 
+       CurrentTime[CurrentProc] - event->tso->gran.blockedat;
+      do_the_startthread(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case StartThread:
+      do_the_startthread(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case MoveThread:
+      do_the_movethread(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case MoveSpark:
+      do_the_movespark(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    case FindWork:
+      do_the_findwork(event);
+      goto next_thread;             /* handle next event in event queue  */
+      
+    default:
+      barf("Illegal event type %u\n", event->evttype);
+    }  /* switch */
+    
+    /* This point was scheduler_loop in the old RTS */
+
+    IF_DEBUG(gran, belch("GRAN: after main switch"));
+
+    TimeOfLastEvent = CurrentTime[CurrentProc];
+    TimeOfNextEvent = get_time_of_next_event();
+    IgnoreEvents=(TimeOfNextEvent==0); // HWL HACK
+    // CurrentTSO = ThreadQueueHd;
+
+    IF_DEBUG(gran, belch("GRAN: time of next event is: %ld", 
+                        TimeOfNextEvent));
+
+    if (RtsFlags.GranFlags.Light) 
+      GranSimLight_leave_system(event, &ActiveTSO); 
+
+    EndOfTimeSlice = CurrentTime[CurrentProc]+RtsFlags.GranFlags.time_slice;
+
+    IF_DEBUG(gran, 
+            belch("GRAN: end of time-slice is %#lx", EndOfTimeSlice));
+
+    /* in a GranSim setup the TSO stays on the run queue */
+    t = CurrentTSO;
+    /* Take a thread from the run queue. */
+    t = POP_RUN_QUEUE(); // take_off_run_queue(t);
+
+    IF_DEBUG(gran, 
+            fprintf(stderr, "GRAN: About to run current thread, which is\n");
+            G_TSO(t,5))
+
+    context_switch = 0; // turned on via GranYield, checking events and time slice
+
+    IF_DEBUG(gran, 
+            DumpGranEvent(GR_SCHEDULE, t));
+
+    procStatus[CurrentProc] = Busy;
+
 #elif defined(PAR)
+
+    if (PendingFetches != END_BF_QUEUE) {
+        processFetches();
+    }
+
     /* ToDo: phps merge with spark activation above */
     /* check whether we have local work and send requests if we have none */
     if (run_queue_hd == END_TSO_QUEUE) {  /* no runnable threads */
       /* :-[  no local threads => look out for local sparks */
+      /* the spark pool for the current PE */
+      pool = &(MainRegTable.rSparks); // generalise to cap = &MainRegTable
       if (advisory_thread_count < RtsFlags.ParFlags.maxThreads &&
-         (pending_sparks_hd[REQUIRED_POOL] < pending_sparks_tl[REQUIRED_POOL] ||
-          pending_sparks_hd[ADVISORY_POOL] < pending_sparks_tl[ADVISORY_POOL])) {
+         pool->hd < pool->tl) {
        /* 
         * ToDo: add GC code check that we really have enough heap afterwards!!
         * Old comment:
@@ -528,18 +752,18 @@ schedule( void )
        spark = findSpark();                /* get a spark */
        if (spark != (rtsSpark) NULL) {
          tso = activateSpark(spark);       /* turn the spark into a thread */
-         IF_PAR_DEBUG(verbose,
-                      belch("== [%x] schedule: Created TSO %p (%d); %d threads active",
-                            mytid, tso, tso->id, advisory_thread_count));
+         IF_PAR_DEBUG(schedule,
+                      belch("==== schedule: Created TSO %d (%p); %d threads active",
+                            tso->id, tso, advisory_thread_count));
 
          if (tso==END_TSO_QUEUE) { /* failed to activate spark->back to loop */
-           belch("^^ failed to activate spark");
+           belch("==^^ failed to activate spark");
            goto next_thread;
          }               /* otherwise fall through & pick-up new tso */
        } else {
          IF_PAR_DEBUG(verbose,
-                      belch("^^ no local sparks (spark pool contains only NFs: %d)", 
-                            spark_queue_len(ADVISORY_POOL)));
+                      belch("==^^ no local sparks (spark pool contains only NFs: %d)", 
+                            spark_queue_len(pool)));
          goto next_thread;
        }
       } else  
@@ -574,25 +798,27 @@ schedule( void )
     /* Now we are sure that we have some work available */
     ASSERT(run_queue_hd != END_TSO_QUEUE);
     /* Take a thread from the run queue, if we have work */
-    t = take_off_run_queue(END_TSO_QUEUE);
+    t = POP_RUN_QUEUE();  // take_off_run_queue(END_TSO_QUEUE);
 
     /* ToDo: write something to the log-file
     if (RTSflags.ParFlags.granSimStats && !sameThread)
         DumpGranEvent(GR_SCHEDULE, RunnableThreadsHd);
-    */
 
     CurrentTSO = t;
+    */
+    /* the spark pool for the current PE */
+    pool = &(MainRegTable.rSparks); // generalise to cap = &MainRegTable
 
-    IF_DEBUG(scheduler, belch("--^^ %d sparks on [%#x] (hd=%x; tl=%x; lim=%x)", 
-                             spark_queue_len(ADVISORY_POOL), CURRENT_PROC,
-                             pending_sparks_hd[ADVISORY_POOL], 
-                             pending_sparks_tl[ADVISORY_POOL], 
-                             pending_sparks_lim[ADVISORY_POOL]));
+    IF_DEBUG(scheduler, belch("--^^ %d sparks on [%#x] (hd=%x; tl=%x; base=%x, lim=%x)", 
+                             spark_queue_len(pool), 
+                             CURRENT_PROC,
+                             pool->hd, pool->tl, pool->base, pool->lim));
 
     IF_DEBUG(scheduler, belch("--== %d threads on [%#x] (hd=%x; tl=%x)", 
                              run_queue_len(), CURRENT_PROC,
                              run_queue_hd, run_queue_tl));
 
+#if 0
     if (t != LastTSO) {
       /* 
         we are running a different TSO, so write a schedule event to log file
@@ -605,11 +831,12 @@ schedule( void )
                       GR_SCHEDULE, t, (StgClosure *)NULL, 0, 0);
       
     }
-
+#endif
 #else /* !GRAN && !PAR */
   
     /* grab a thread from the run queue
      */
+    ASSERT(run_queue_hd != END_TSO_QUEUE);
     t = POP_RUN_QUEUE();
     IF_DEBUG(sanity,checkTSO(t));
 
@@ -627,21 +854,27 @@ schedule( void )
     
     cap->rCurrentTSO = t;
     
-    /* set the context_switch flag
+    /* context switches are now initiated by the timer signal, unless
+     * the user specified "context switch as often as possible", with
+     * +RTS -C0
      */
-    if (run_queue_hd == END_TSO_QUEUE)
-      context_switch = 0;
+    if (RtsFlags.ConcFlags.ctxtSwitchTicks == 0
+       && (run_queue_hd != END_TSO_QUEUE
+           || blocked_queue_hd != END_TSO_QUEUE
+           || sleeping_queue != END_TSO_QUEUE))
+       context_switch = 1;
     else
-      context_switch = 1;
+       context_switch = 0;
 
     RELEASE_LOCK(&sched_mutex);
-    
-    IF_DEBUG(scheduler,sched_belch("running thread %d", t->id));
+
+    IF_DEBUG(scheduler, sched_belch("-->> Running TSO %ld (%p) %s ...", 
+                             t->id, t, whatNext_strs[t->what_next]));
 
     /* +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
     /* Run the current thread 
      */
-    switch (cap->rCurrentTSO->whatNext) {
+    switch (cap->rCurrentTSO->what_next) {
     case ThreadKilled:
     case ThreadComplete:
       /* Thread already finished, return to scheduler. */
@@ -653,21 +886,18 @@ schedule( void )
     case ThreadRunGHC:
       ret = StgRun((StgFunPtr) stg_returnToStackTop, cap);
       break;
-    case ThreadEnterHugs:
-#ifdef INTERPRETER
+    case ThreadEnterInterp:
+#ifdef GHCI
       {
-         StgClosure* c;
-        IF_DEBUG(scheduler,sched_belch("entering Hugs"));
-        c = (StgClosure *)(cap->rCurrentTSO->sp[0]);
-        cap->rCurrentTSO->sp += 1;
-        ret = enter(cap,c);
+        IF_DEBUG(scheduler,sched_belch("entering interpreter"));
+        ret = interpretBCO(cap);
          break;
       }
 #else
       barf("Panic: entered a BCO but no bytecode interpreter in this build");
 #endif
     default:
-      barf("schedule: invalid whatNext field");
+      barf("schedule: invalid what_next field");
     }
     /* +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ */
     
@@ -680,30 +910,44 @@ schedule( void )
 
 #ifdef SMP
     IF_DEBUG(scheduler,fprintf(stderr,"scheduler (task %ld): ", pthread_self()););
-#else
+#elif !defined(GRAN) && !defined(PAR)
     IF_DEBUG(scheduler,fprintf(stderr,"scheduler: "););
 #endif
     t = cap->rCurrentTSO;
     
+#if defined(PAR)
+    /* HACK 675: if the last thread didn't yield, make sure to print a 
+       SCHEDULE event to the log file when StgRunning the next thread, even
+       if it is the same one as before */
+    LastTSO = t; //(ret == ThreadBlocked) ? END_TSO_QUEUE : t; 
+    TimeOfLastYield = CURRENT_TIME;
+#endif
+
     switch (ret) {
     case HeapOverflow:
       /* make all the running tasks block on a condition variable,
        * maybe set context_switch and wait till they all pile in,
        * then have them wait on a GC condition variable.
        */
-      IF_DEBUG(scheduler,belch("thread %ld stopped: HeapOverflow", t->id));
+      IF_DEBUG(scheduler,belch("--<< thread %ld (%p; %s) stopped: HeapOverflow", 
+                              t->id, t, whatNext_strs[t->what_next]));
       threadPaused(t);
+#if defined(GRAN)
+      ASSERT(!is_on_queue(t,CurrentProc));
+#endif
       
       ready_to_gc = rtsTrue;
       context_switch = 1;              /* stop other threads ASAP */
       PUSH_ON_RUN_QUEUE(t);
+      /* actual GC is done at the end of the while loop */
       break;
       
     case StackOverflow:
+      IF_DEBUG(scheduler,belch("--<< thread %ld (%p; %s) stopped, StackOverflow", 
+                              t->id, t, whatNext_strs[t->what_next]));
       /* just adjust the stack for this thread, then pop it back
        * on the run queue.
        */
-      IF_DEBUG(scheduler,belch("thread %ld stopped, StackOverflow", t->id));
       threadPaused(t);
       { 
        StgMainThread *m;
@@ -739,36 +983,93 @@ schedule( void )
        * GC is finished.
        */
       IF_DEBUG(scheduler,
-              if (t->whatNext == ThreadEnterHugs) {
-                /* ToDo: or maybe a timer expired when we were in Hugs?
-                 * or maybe someone hit ctrl-C
-                 */
-                belch("thread %ld stopped to switch to Hugs", t->id);
-              } else {
-                belch("thread %ld stopped, yielding", t->id);
-              }
-              );
+               if (t->what_next == ThreadEnterInterp) {
+                  /* ToDo: or maybe a timer expired when we were in Hugs?
+                   * or maybe someone hit ctrl-C
+                    */
+                   belch("--<< thread %ld (%p; %s) stopped to switch to Hugs", 
+                        t->id, t, whatNext_strs[t->what_next]);
+               } else {
+                   belch("--<< thread %ld (%p; %s) stopped, yielding", 
+                        t->id, t, whatNext_strs[t->what_next]);
+               }
+               );
+
       threadPaused(t);
+
+      IF_DEBUG(sanity,
+              //belch("&& Doing sanity check on yielding TSO %ld.", t->id);
+              checkTSO(t));
+      ASSERT(t->link == END_TSO_QUEUE);
+#if defined(GRAN)
+      ASSERT(!is_on_queue(t,CurrentProc));
+
+      IF_DEBUG(sanity,
+              //belch("&& Doing sanity check on all ThreadQueues (and their TSOs).");
+              checkThreadQsSanity(rtsTrue));
+#endif
       APPEND_TO_RUN_QUEUE(t);
+#if defined(GRAN)
+      /* add a ContinueThread event to actually process the thread */
+      new_event(CurrentProc, CurrentProc, CurrentTime[CurrentProc],
+               ContinueThread,
+               t, (StgClosure*)NULL, (rtsSpark*)NULL);
+      IF_GRAN_DEBUG(bq, 
+              belch("GRAN: eventq and runnableq after adding yielded thread to queue again:");
+              G_EVENTQ(0);
+              G_CURR_THREADQ(0))
+#endif /* GRAN */
       break;
       
     case ThreadBlocked:
 #if defined(GRAN)
-# error ToDo: implement GranSim scheduler
+      IF_DEBUG(scheduler,
+              belch("--<< thread %ld (%p; %s) stopped, blocking on node %p [PE %d] with BQ: ", 
+                              t->id, t, whatNext_strs[t->what_next], t->block_info.closure, (t->block_info.closure==(StgClosure*)NULL ? 99 : where_is(t->block_info.closure)));
+              if (t->block_info.closure!=(StgClosure*)NULL) print_bq(t->block_info.closure));
+
+      // ??? needed; should emit block before
+      IF_DEBUG(gran, 
+              DumpGranEvent(GR_DESCHEDULE, t)); 
+      prune_eventq(t, (StgClosure *)NULL); // prune ContinueThreads for t
+      /*
+       ngoq Dogh!
+      ASSERT(procStatus[CurrentProc]==Busy || 
+             ((procStatus[CurrentProc]==Fetching) && 
+             (t->block_info.closure!=(StgClosure*)NULL)));
+      if (run_queue_hds[CurrentProc] == END_TSO_QUEUE &&
+         !(!RtsFlags.GranFlags.DoAsyncFetch &&
+           procStatus[CurrentProc]==Fetching)) 
+       procStatus[CurrentProc] = Idle;
+      */
 #elif defined(PAR)
       IF_DEBUG(par, 
               DumpGranEvent(GR_DESCHEDULE, t)); 
-#else
-#endif
+
+      /* Send a fetch (if BlockedOnGA) and dump event to log file */
+      blockThread(t);
+
+      IF_DEBUG(scheduler,
+              belch("--<< thread %ld (%p; %s) stopped, blocking on node %p with BQ: ", 
+                              t->id, t, whatNext_strs[t->what_next], t->block_info.closure);
+              if (t->block_info.closure!=(StgClosure*)NULL) print_bq(t->block_info.closure));
+
+#else /* !GRAN */
       /* don't need to do anything.  Either the thread is blocked on
        * I/O, in which case we'll have called addToBlockedQueue
        * previously, or it's blocked on an MVar or Blackhole, in which
        * case it'll be on the relevant queue already.
        */
       IF_DEBUG(scheduler,
-              fprintf(stderr, "thread %d stopped, ", t->id);
+              fprintf(stderr, "--<< thread %d (%p) stopped: ", t->id, t);
               printThreadBlockage(t);
               fprintf(stderr, "\n"));
+
+      /* Only for dumping event to log file 
+        ToDo: do I need this in GranSim, too?
+      blockThread(t);
+      */
+#endif
       threadPaused(t);
       break;
       
@@ -778,10 +1079,12 @@ schedule( void )
        * more main threads, we probably need to stop all the tasks until
        * we get a new one.
        */
-      IF_DEBUG(scheduler,belch("thread %ld finished", t->id));
-      t->whatNext = ThreadComplete;
+      /* We also end up here if the thread kills itself with an
+       * uncaught exception, see Exception.hc.
+       */
+      IF_DEBUG(scheduler,belch("--++ thread %d (%p) finished", t->id, t));
 #if defined(GRAN)
-      // ToDo: endThread(t, CurrentProc); // clean-up the thread
+      endThread(t, CurrentProc); // clean-up the thread
 #elif defined(PAR)
       advisory_thread_count--;
       if (RtsFlags.ParFlags.ParStats.Full) 
@@ -790,7 +1093,7 @@ schedule( void )
       break;
       
     default:
-      barf("doneThread: invalid thread return code");
+      barf("schedule: invalid thread return code %d", (int)ret);
     }
     
 #ifdef SMP
@@ -813,11 +1116,21 @@ schedule( void )
 #ifdef SMP
       IF_DEBUG(scheduler,sched_belch("doing GC"));
 #endif
-      GarbageCollect(GetRoots);
+      GarbageCollect(GetRoots,rtsFalse);
       ready_to_gc = rtsFalse;
 #ifdef SMP
       pthread_cond_broadcast(&gc_pending_cond);
 #endif
+#if defined(GRAN)
+      /* add a ContinueThread event to continue execution of current thread */
+      new_event(CurrentProc, CurrentProc, CurrentTime[CurrentProc],
+               ContinueThread,
+               t, (StgClosure*)NULL, (rtsSpark*)NULL);
+      IF_GRAN_DEBUG(bq, 
+              fprintf(stderr, "GRAN: eventq and runnableq after Garbage collection:\n");
+              G_EVENTQ(0);
+              G_CURR_THREADQ(0))
+#endif /* GRAN */
     }
 #if defined(GRAN)
   next_thread:
@@ -839,19 +1152,29 @@ schedule( void )
   } /* end of while(1) */
 }
 
-/* A hack for Hugs concurrency support.  Needs sanitisation (?) */
+/* ---------------------------------------------------------------------------
+ * deleteAllThreads():  kill all the live threads.
+ *
+ * This is used when we catch a user interrupt (^C), before performing
+ * any necessary cleanups and running finalizers.
+ * ------------------------------------------------------------------------- */
+   
 void deleteAllThreads ( void )
 {
   StgTSO* t;
-  IF_DEBUG(scheduler,sched_belch("deleteAllThreads()"));
+  IF_DEBUG(scheduler,sched_belch("deleting all threads"));
   for (t = run_queue_hd; t != END_TSO_QUEUE; t = t->link) {
-    deleteThread(t);
+      deleteThread(t);
   }
   for (t = blocked_queue_hd; t != END_TSO_QUEUE; t = t->link) {
-    deleteThread(t);
+      deleteThread(t);
+  }
+  for (t = sleeping_queue; t != END_TSO_QUEUE; t = t->link) {
+      deleteThread(t);
   }
   run_queue_hd = run_queue_tl = END_TSO_QUEUE;
   blocked_queue_hd = blocked_queue_tl = END_TSO_QUEUE;
+  sleeping_queue = END_TSO_QUEUE;
 }
 
 /* startThread and  insertThread are now in GranSim.c -- HWL */
@@ -882,7 +1205,7 @@ suspendThread( Capability *cap )
   ACQUIRE_LOCK(&sched_mutex);
 
   IF_DEBUG(scheduler,
-          sched_belch("thread %d did a _ccall_gc\n", cap->rCurrentTSO->id));
+          sched_belch("thread %d did a _ccall_gc", cap->rCurrentTSO->id));
 
   threadPaused(cap->rCurrentTSO);
   cap->rCurrentTSO->link = suspended_ccalling_threads;
@@ -921,6 +1244,7 @@ resumeThread( StgInt tok )
   if (tso == END_TSO_QUEUE) {
     barf("resumeThread: thread not found");
   }
+  tso->link = END_TSO_QUEUE;
 
 #ifdef SMP
   while (free_capabilities == NULL) {
@@ -974,10 +1298,12 @@ int cmp_thread(const StgTSO *tso1, const StgTSO *tso2)
 
    createGenThread() and createIOThread() (in SchedAPI.h) are
    convenient packaged versions of this function.
+
+   currently pri (priority) is only used in a GRAN setup -- HWL
    ------------------------------------------------------------------------ */
 //@cindex createThread
 #if defined(GRAN)
-/* currently pri (priority) is only used in a GRAN setup -- HWL */
+/*   currently pri (priority) is only used in a GRAN setup -- HWL */
 StgTSO *
 createThread(nat stack_size, StgInt pri)
 {
@@ -998,6 +1324,7 @@ static StgTSO *
 createThread_(nat size, rtsBool have_lock)
 {
 #endif
+
     StgTSO *tso;
     nat stack_size;
 
@@ -1029,11 +1356,11 @@ createThread_(nat size, rtsBool have_lock)
   tso = (StgTSO *)allocate(size);
   TICK_ALLOC_TSO(size-TSO_STRUCT_SIZEW, 0);
 
-  SET_HDR(tso, &TSO_info, CCS_MAIN);
+  SET_HDR(tso, &stg_TSO_info, CCS_SYSTEM);
 #if defined(GRAN)
   SET_GRAN_HDR(tso, ThisPE);
 #endif
-  tso->whatNext     = ThreadEnterGHC;
+  tso->what_next     = ThreadEnterGHC;
 
   /* tso->id needs to be unique.  For now we use a heavyweight mutex to
    * protect the increment operation on next_thread_id.
@@ -1046,7 +1373,6 @@ createThread_(nat size, rtsBool have_lock)
   tso->why_blocked  = NotBlocked;
   tso->blocked_exceptions = NULL;
 
-  tso->splim        = (P_)&(tso->stack) + RESERVED_STACK_WORDS;
   tso->stack_size   = stack_size;
   tso->max_stack_size = round_to_mblocks(RtsFlags.GcFlags.maxStkSize) 
                               - TSO_STRUCT_SIZEW;
@@ -1058,12 +1384,9 @@ createThread_(nat size, rtsBool have_lock)
 
   /* put a stop frame on the stack */
   tso->sp -= sizeofW(StgStopFrame);
-  SET_HDR((StgClosure*)tso->sp,(StgInfoTable *)&stg_stop_thread_info,CCS_MAIN);
+  SET_HDR((StgClosure*)tso->sp,(StgInfoTable *)&stg_stop_thread_info,CCS_SYSTEM);
   tso->su = (StgUpdateFrame*)tso->sp;
 
-  IF_DEBUG(scheduler,belch("---- Initialised TSO %ld (%p), stack size = %lx words", 
-                          tso->id, tso, tso->stack_size));
-
   // ToDo: check this
 #if defined(GRAN)
   tso->link = END_TSO_QUEUE;
@@ -1075,18 +1398,20 @@ createThread_(nat size, rtsBool have_lock)
    */
 #endif
 
+#if defined(GRAN) || defined(PAR)
+  DumpGranEvent(GR_START,tso);
+#endif
+
   /* Link the new thread on the global thread list.
    */
-#if defined(GRAN)
-#error ToDo
-#else
   tso->global_link = all_threads;
   all_threads = tso;
-#endif
 
 #if defined(GRAN)
   tso->gran.pri = pri;
+# if defined(DEBUG)
   tso->gran.magic = TSO_MAGIC; // debugging only
+# endif
   tso->gran.sparkname   = 0;
   tso->gran.startedat   = CURRENT_TIME; 
   tso->gran.exported    = 0;
@@ -1107,6 +1432,9 @@ createThread_(nat size, rtsBool have_lock)
 
   IF_DEBUG(gran,printTSO(tso));
 #elif defined(PAR)
+# if defined(DEBUG)
+  tso->par.magic = TSO_MAGIC; // debugging only
+# endif
   tso->par.sparkname   = 0;
   tso->par.startedat   = CURRENT_TIME; 
   tso->par.exported    = 0;
@@ -1129,10 +1457,56 @@ createThread_(nat size, rtsBool have_lock)
   globalGranStats.tot_sq_probes++;
 #endif 
 
+#if defined(GRAN)
+  IF_GRAN_DEBUG(pri,
+               belch("==__ schedule: Created TSO %d (%p);",
+                     CurrentProc, tso, tso->id));
+#elif defined(PAR)
+    IF_PAR_DEBUG(verbose,
+                belch("==__ schedule: Created TSO %d (%p); %d threads active",
+                      tso->id, tso, advisory_thread_count));
+#else
   IF_DEBUG(scheduler,sched_belch("created thread %ld, stack size = %lx words", 
                                 tso->id, tso->stack_size));
+#endif    
+  return tso;
+}
+
+/*
+  Turn a spark into a thread.
+  ToDo: fix for SMP (needs to acquire SCHED_MUTEX!)
+*/
+#if defined(PAR)
+//@cindex activateSpark
+StgTSO *
+activateSpark (rtsSpark spark) 
+{
+  StgTSO *tso;
+  
+  ASSERT(spark != (rtsSpark)NULL);
+  tso = createThread_(RtsFlags.GcFlags.initialStkSize, rtsTrue);
+  if (tso!=END_TSO_QUEUE) {
+    pushClosure(tso,spark);
+    PUSH_ON_RUN_QUEUE(tso);
+    advisory_thread_count++;
+
+    if (RtsFlags.ParFlags.ParStats.Full) {
+      //ASSERT(run_queue_hd == END_TSO_QUEUE); // I think ...
+      IF_PAR_DEBUG(verbose,
+                  belch("==^^ activateSpark: turning spark of closure %p (%s) into a thread",
+                        (StgClosure *)spark, info_type((StgClosure *)spark)));
+    }
+  } else {
+    barf("activateSpark: Cannot create TSO");
+  }
+  // ToDo: fwd info on local/global spark to thread -- HWL
+  // tso->gran.exported =  spark->exported;
+  // tso->gran.locked =   !spark->global;
+  // tso->gran.sparkname = spark->name;
+
   return tso;
 }
+#endif
 
 /* ---------------------------------------------------------------------------
  * scheduleThread()
@@ -1147,6 +1521,11 @@ createThread_(nat size, rtsBool have_lock)
 void
 scheduleThread(StgTSO *tso)
 {
+  if (tso==END_TSO_QUEUE){    
+    schedule();
+    return;
+  }
+
   ACQUIRE_LOCK(&sched_mutex);
 
   /* Put the new thread on the head of the runnable queue.  The caller
@@ -1157,7 +1536,9 @@ scheduleThread(StgTSO *tso)
   PUSH_ON_RUN_QUEUE(tso);
   THREAD_RUNNABLE();
 
+#if 0
   IF_DEBUG(scheduler,printTSO(tso));
+#endif
   RELEASE_LOCK(&sched_mutex);
 }
 
@@ -1169,12 +1550,11 @@ scheduleThread(StgTSO *tso)
  *  KH @ 25/10/99
  * ------------------------------------------------------------------------ */
 
-#ifdef SMP
-static void *
+#if defined(PAR) || defined(SMP)
+void *
 taskStart( void *arg STG_UNUSED )
 {
-  schedule();
-  return NULL;
+  rts_evalNothing(NULL);
 }
 #endif
 
@@ -1213,12 +1593,14 @@ initScheduler(void)
     blocked_queue_hds[i]  = END_TSO_QUEUE;
     blocked_queue_tls[i]  = END_TSO_QUEUE;
     ccalling_threadss[i]  = END_TSO_QUEUE;
+    sleeping_queue        = END_TSO_QUEUE;
   }
 #else
   run_queue_hd      = END_TSO_QUEUE;
   run_queue_tl      = END_TSO_QUEUE;
   blocked_queue_hd  = END_TSO_QUEUE;
   blocked_queue_tl  = END_TSO_QUEUE;
+  sleeping_queue    = END_TSO_QUEUE;
 #endif 
 
   suspended_ccalling_threads  = END_TSO_QUEUE;
@@ -1229,7 +1611,13 @@ initScheduler(void)
   context_switch = 0;
   interrupted    = 0;
 
-  enteredCAFs = END_CAF_LIST;
+  RtsFlags.ConcFlags.ctxtSwitchTicks =
+      RtsFlags.ConcFlags.ctxtSwitchTime / TICK_MILLISECS;
+
+#ifdef INTERPRETER
+  ecafList = END_ECAF_LIST;
+  clearECafTable();
+#endif
 
   /* Install the SIGHUP handler */
 #ifdef SMP
@@ -1344,7 +1732,7 @@ exitScheduler( void )
    -------------------------------------------------------------------------- */
 
 /* -----------------------------------------------------------------------------
- * waitThread is the external interface for running a new computataion
+ * waitThread is the external interface for running a new computation
  * and waiting for the result.
  *
  * In the non-SMP case, we create a new main thread, push it on the 
@@ -1359,6 +1747,39 @@ exitScheduler( void )
  * will be in the main_thread struct.
  * -------------------------------------------------------------------------- */
 
+int 
+howManyThreadsAvail ( void )
+{
+   int i = 0;
+   StgTSO* q;
+   for (q = run_queue_hd; q != END_TSO_QUEUE; q = q->link)
+      i++;
+   for (q = blocked_queue_hd; q != END_TSO_QUEUE; q = q->link)
+      i++;
+   for (q = sleeping_queue; q != END_TSO_QUEUE; q = q->link)
+      i++;
+   return i;
+}
+
+void
+finishAllThreads ( void )
+{
+   do {
+      while (run_queue_hd != END_TSO_QUEUE) {
+         waitThread ( run_queue_hd, NULL );
+      }
+      while (blocked_queue_hd != END_TSO_QUEUE) {
+         waitThread ( blocked_queue_hd, NULL );
+      }
+      while (sleeping_queue != END_TSO_QUEUE) {
+         waitThread ( blocked_queue_hd, NULL );
+      }
+   } while 
+      (blocked_queue_hd != END_TSO_QUEUE || 
+       run_queue_hd     != END_TSO_QUEUE ||
+       sleeping_queue   != END_TSO_QUEUE);
+}
+
 SchedulerStatus
 waitThread(StgTSO *tso, /*out*/StgClosure **ret)
 {
@@ -1386,6 +1807,13 @@ waitThread(StgTSO *tso, /*out*/StgClosure **ret)
   do {
     pthread_cond_wait(&m->wakeup, &sched_mutex);
   } while (m->stat == NoStatus);
+#elif defined(GRAN)
+  /* GranSim specific init */
+  CurrentTSO = m->tso;                // the TSO to run
+  procStatus[MainProc] = Busy;        // status of main PE
+  CurrentProc = MainProc;             // PE to run it on
+
+  schedule();
 #else
   schedule();
   ASSERT(m->stat != NoStatus);
@@ -1515,6 +1943,7 @@ take_off_run_queue(StgTSO *tso) {
 
         - all the threads on the runnable queue
         - all the threads on the blocked queue
+        - all the threads on the sleeping queue
        - all the thread currently executing a _ccall_GC
         - all the "main threads"
      
@@ -1550,22 +1979,37 @@ static void GetRoots(void)
   markEventQueue();
 
 #else /* !GRAN */
-  run_queue_hd      = (StgTSO *)MarkRoot((StgClosure *)run_queue_hd);
-  run_queue_tl      = (StgTSO *)MarkRoot((StgClosure *)run_queue_tl);
+  if (run_queue_hd != END_TSO_QUEUE) {
+    ASSERT(run_queue_tl != END_TSO_QUEUE);
+    run_queue_hd      = (StgTSO *)MarkRoot((StgClosure *)run_queue_hd);
+    run_queue_tl      = (StgTSO *)MarkRoot((StgClosure *)run_queue_tl);
+  }
+
+  if (blocked_queue_hd != END_TSO_QUEUE) {
+    ASSERT(blocked_queue_tl != END_TSO_QUEUE);
+    blocked_queue_hd  = (StgTSO *)MarkRoot((StgClosure *)blocked_queue_hd);
+    blocked_queue_tl  = (StgTSO *)MarkRoot((StgClosure *)blocked_queue_tl);
+  }
 
-  blocked_queue_hd  = (StgTSO *)MarkRoot((StgClosure *)blocked_queue_hd);
-  blocked_queue_tl  = (StgTSO *)MarkRoot((StgClosure *)blocked_queue_tl);
+  if (sleeping_queue != END_TSO_QUEUE) {
+    sleeping_queue  = (StgTSO *)MarkRoot((StgClosure *)sleeping_queue);
+  }
 #endif 
 
   for (m = main_threads; m != NULL; m = m->link) {
     m->tso = (StgTSO *)MarkRoot((StgClosure *)m->tso);
   }
-  suspended_ccalling_threads = 
-    (StgTSO *)MarkRoot((StgClosure *)suspended_ccalling_threads);
+  if (suspended_ccalling_threads != END_TSO_QUEUE)
+    suspended_ccalling_threads = 
+      (StgTSO *)MarkRoot((StgClosure *)suspended_ccalling_threads);
 
 #if defined(SMP) || defined(PAR) || defined(GRAN)
   markSparkQueue();
 #endif
+
+#if defined(GHCI)
+  markCafs();
+#endif
 }
 
 /* -----------------------------------------------------------------------------
@@ -1586,7 +2030,13 @@ void (*extra_roots)(void);
 void
 performGC(void)
 {
-  GarbageCollect(GetRoots);
+  GarbageCollect(GetRoots,rtsFalse);
+}
+
+void
+performMajorGC(void)
+{
+  GarbageCollect(GetRoots,rtsTrue);
 }
 
 static void
@@ -1601,7 +2051,7 @@ performGCWithRoots(void (*get_roots)(void))
 {
   extra_roots = get_roots;
 
-  GarbageCollect(AllRoots);
+  GarbageCollect(AllRoots,rtsFalse);
 }
 
 /* -----------------------------------------------------------------------------
@@ -1622,11 +2072,14 @@ threadStackOverflow(StgTSO *tso)
 
   IF_DEBUG(sanity,checkTSO(tso));
   if (tso->stack_size >= tso->max_stack_size) {
-#if 0
-    /* If we're debugging, just print out the top of the stack */
-    printStackChunk(tso->sp, stg_min(tso->stack+tso->stack_size, 
-                                    tso->sp+64));
-#endif
+
+    IF_DEBUG(gc,
+            belch("@@ threadStackOverflow of TSO %d (%p): stack too large (now %ld; max is %ld",
+                  tso->id, tso, tso->stack_size, tso->max_stack_size);
+            /* If we're debugging, just print out the top of the stack */
+            printStackChunk(tso->sp, stg_min(tso->stack+tso->stack_size, 
+                                             tso->sp+64)));
+
 #ifdef INTERPRETER
     fprintf(stderr, "fatal: stack overflow in Hugs; aborting\n" );
     exit(1);
@@ -1662,7 +2115,6 @@ threadStackOverflow(StgTSO *tso)
   diff = (P_)new_sp - (P_)tso->sp; /* In *words* */
   dest->su    = (StgUpdateFrame *) ((P_)dest->su + diff);
   dest->sp    = new_sp;
-  dest->splim = (P_)dest->splim + (nat)((P_)dest - (P_)tso);
   dest->stack_size = new_stack_size;
        
   /* and relocate the update frame list */
@@ -1675,13 +2127,20 @@ threadStackOverflow(StgTSO *tso)
    * of the stack, so we don't attempt to scavenge any part of the
    * dead TSO's stack.
    */
-  tso->whatNext = ThreadRelocated;
+  tso->what_next = ThreadRelocated;
   tso->link = dest;
   tso->sp = (P_)&(tso->stack[tso->stack_size]);
   tso->su = (StgUpdateFrame *)tso->sp;
   tso->why_blocked = NotBlocked;
   dest->mut_link = NULL;
 
+  IF_PAR_DEBUG(verbose,
+              belch("@@ threadStackOverflow of TSO %d (now at %p): stack size increased to %ld",
+                    tso->id, tso, tso->stack_size);
+              /* If we're debugging, just print out the top of the stack */
+              printStackChunk(tso->sp, stg_min(tso->stack+tso->stack_size, 
+                                               tso->sp+64)));
+  
   IF_DEBUG(sanity,checkTSO(tso));
 #if 0
   IF_DEBUG(scheduler,printTSO(dest));
@@ -1697,8 +2156,6 @@ threadStackOverflow(StgTSO *tso)
    Wake up a queue that was blocked on some resource.
    ------------------------------------------------------------------------ */
 
-/* ToDo: check push_on_run_queue vs. PUSH_ON_RUN_QUEUE */
-
 #if defined(GRAN)
 static inline void
 unblockCount ( StgBlockingQueueElement *bqe, StgClosure *node )
@@ -1712,7 +2169,7 @@ unblockCount ( StgBlockingQueueElement *bqe, StgClosure *node )
      update blocked and fetch time (depending on type of the orig closure) */
   if (RtsFlags.ParFlags.ParStats.Full) {
     DumpRawGranEvent(CURRENT_PROC, CURRENT_PROC, 
-                    GR_RESUMEQ, ((StgTSO *)bqe), ((StgTSO *)bqe)->block_info.closure,
+                    GR_RESUME, ((StgTSO *)bqe), ((StgTSO *)bqe)->block_info.closure,
                     0, 0 /* spark_queue_len(ADVISORY_POOL) */);
 
     switch (get_itbl(node)->type) {
@@ -1735,63 +2192,41 @@ unblockCount ( StgBlockingQueueElement *bqe, StgClosure *node )
 static StgBlockingQueueElement *
 unblockOneLocked(StgBlockingQueueElement *bqe, StgClosure *node)
 {
-    StgBlockingQueueElement *next;
+    StgTSO *tso;
     PEs node_loc, tso_loc;
 
     node_loc = where_is(node); // should be lifted out of loop
     tso = (StgTSO *)bqe;  // wastes an assignment to get the type right
-    tso_loc = where_is(tso);
+    tso_loc = where_is((StgClosure *)tso);
     if (IS_LOCAL_TO(PROCS(node),tso_loc)) { // TSO is local
       /* !fake_fetch => TSO is on CurrentProc is same as IS_LOCAL_TO */
       ASSERT(CurrentProc!=node_loc || tso_loc==CurrentProc);
-      bq_processing_time += RtsFlags.GranFlags.Costs.lunblocktime;
+      CurrentTime[CurrentProc] += RtsFlags.GranFlags.Costs.lunblocktime;
       // insertThread(tso, node_loc);
-      new_event(tso_loc, tso_loc,
-               CurrentTime[CurrentProc]+bq_processing_time,
+      new_event(tso_loc, tso_loc, CurrentTime[CurrentProc],
                ResumeThread,
                tso, node, (rtsSpark*)NULL);
       tso->link = END_TSO_QUEUE; // overwrite link just to be sure 
       // len_local++;
       // len++;
     } else { // TSO is remote (actually should be FMBQ)
-      bq_processing_time += RtsFlags.GranFlags.Costs.mpacktime;
-      bq_processing_time += RtsFlags.GranFlags.Costs.gunblocktime;
-      new_event(tso_loc, CurrentProc, 
-               CurrentTime[CurrentProc]+bq_processing_time+
-               RtsFlags.GranFlags.Costs.latency,
+      CurrentTime[CurrentProc] += RtsFlags.GranFlags.Costs.mpacktime +
+                                  RtsFlags.GranFlags.Costs.gunblocktime +
+                                 RtsFlags.GranFlags.Costs.latency;
+      new_event(tso_loc, CurrentProc, CurrentTime[CurrentProc],
                UnblockThread,
                tso, node, (rtsSpark*)NULL);
       tso->link = END_TSO_QUEUE; // overwrite link just to be sure 
-      bq_processing_time += RtsFlags.GranFlags.Costs.mtidytime;
       // len++;
-    }      
+    }
     /* the thread-queue-overhead is accounted for in either Resume or UnblockThread */
     IF_GRAN_DEBUG(bq,
-                 fprintf(stderr," %s TSO %d (%p) [PE %d] (blocked_on=%p) (next=%p) ,",
+                 fprintf(stderr," %s TSO %d (%p) [PE %d] (block_info.closure=%p) (next=%p) ,",
                          (node_loc==tso_loc ? "Local" : "Global"), 
-                         tso->id, tso, CurrentProc, tso->blocked_on, tso->link))
-    tso->blocked_on = NULL;
+                         tso->id, tso, CurrentProc, tso->block_info.closure, tso->link));
+    tso->block_info.closure = NULL;
     IF_DEBUG(scheduler,belch("-- Waking up thread %ld (%p)", 
                             tso->id, tso));
-  }
-
-  /* if this is the BQ of an RBH, we have to put back the info ripped out of
-     the closure to make room for the anchor of the BQ */
-  if (next!=END_BQ_QUEUE) {
-    ASSERT(get_itbl(node)->type == RBH && get_itbl(next)->type == CONSTR);
-    /*
-    ASSERT((info_ptr==&RBH_Save_0_info) ||
-          (info_ptr==&RBH_Save_1_info) ||
-          (info_ptr==&RBH_Save_2_info));
-    */
-    /* cf. convertToRBH in RBH.c for writing the RBHSave closure */
-    ((StgRBH *)node)->blocking_queue = ((StgRBHSave *)next)->payload[0];
-    ((StgRBH *)node)->mut_link       = ((StgRBHSave *)next)->payload[1];
-
-    IF_GRAN_DEBUG(bq,
-                 belch("## Filled in RBH_Save for %p (%s) at end of AwBQ",
-                       node, info_type(node)));
-  }
 }
 #elif defined(PAR)
 static StgBlockingQueueElement *
@@ -1824,9 +2259,9 @@ unblockOneLocked(StgBlockingQueueElement *bqe, StgClosure *node)
         see comments on RBHSave closures above */
     case CONSTR:
       /* check that the closure is an RBHSave closure */
-      ASSERT(get_itbl((StgClosure *)bqe) == &RBH_Save_0_info ||
-            get_itbl((StgClosure *)bqe) == &RBH_Save_1_info ||
-            get_itbl((StgClosure *)bqe) == &RBH_Save_2_info);
+      ASSERT(get_itbl((StgClosure *)bqe) == &stg_RBH_Save_0_info ||
+            get_itbl((StgClosure *)bqe) == &stg_RBH_Save_1_info ||
+            get_itbl((StgClosure *)bqe) == &stg_RBH_Save_2_info);
       break;
 
     default:
@@ -1856,14 +2291,14 @@ unblockOneLocked(StgTSO *tso)
 }
 #endif
 
-#if defined(PAR) || defined(GRAN)
-inline StgTSO *
-unblockOne(StgTSO *tso, StgClosure *node)
+#if defined(GRAN) || defined(PAR)
+inline StgBlockingQueueElement *
+unblockOne(StgBlockingQueueElement *bqe, StgClosure *node)
 {
   ACQUIRE_LOCK(&sched_mutex);
-  tso = unblockOneLocked(tso, node);
+  bqe = unblockOneLocked(bqe, node);
   RELEASE_LOCK(&sched_mutex);
-  return tso;
+  return bqe;
 }
 #else
 inline StgTSO *
@@ -1880,11 +2315,9 @@ unblockOne(StgTSO *tso)
 void 
 awakenBlockedQueue(StgBlockingQueueElement *q, StgClosure *node)
 {
-  StgBlockingQueueElement *bqe, *next;
-  StgTSO *tso;
-  PEs node_loc, tso_loc;
-  rtsTime bq_processing_time = 0;
-  nat len = 0, len_local = 0;
+  StgBlockingQueueElement *bqe;
+  PEs node_loc;
+  nat len = 0; 
 
   IF_GRAN_DEBUG(bq, 
                belch("## AwBQ for node %p on PE %d @ %ld by TSO %d (%p): ", \
@@ -1926,21 +2359,38 @@ awakenBlockedQueue(StgBlockingQueueElement *q, StgClosure *node)
     */
     //tso = (StgTSO *)bqe;  // wastes an assignment to get the type right
     //tso_loc = where_is(tso);
+    len++;
     bqe = unblockOneLocked(bqe, node);
   }
 
+  /* if this is the BQ of an RBH, we have to put back the info ripped out of
+     the closure to make room for the anchor of the BQ */
+  if (bqe!=END_BQ_QUEUE) {
+    ASSERT(get_itbl(node)->type == RBH && get_itbl(bqe)->type == CONSTR);
+    /*
+    ASSERT((info_ptr==&RBH_Save_0_info) ||
+          (info_ptr==&RBH_Save_1_info) ||
+          (info_ptr==&RBH_Save_2_info));
+    */
+    /* cf. convertToRBH in RBH.c for writing the RBHSave closure */
+    ((StgRBH *)node)->blocking_queue = (StgBlockingQueueElement *)((StgRBHSave *)bqe)->payload[0];
+    ((StgRBH *)node)->mut_link       = (StgMutClosure *)((StgRBHSave *)bqe)->payload[1];
+
+    IF_GRAN_DEBUG(bq,
+                 belch("## Filled in RBH_Save for %p (%s) at end of AwBQ",
+                       node, info_type(node)));
+  }
+
   /* statistics gathering */
-  /* ToDo: fix counters
   if (RtsFlags.GranFlags.GranSimStats.Global) {
-    globalGranStats.tot_bq_processing_time += bq_processing_time;
+    // globalGranStats.tot_bq_processing_time += bq_processing_time;
     globalGranStats.tot_bq_len += len;      // total length of all bqs awakened
-    globalGranStats.tot_bq_len_local += len_local;  // same for local TSOs only
+    // globalGranStats.tot_bq_len_local += len_local;  // same for local TSOs only
     globalGranStats.tot_awbq++;             // total no. of bqs awakened
   }
   IF_GRAN_DEBUG(bq,
-               fprintf(stderr,"## BQ Stats of %p: [%d entries, %d local] %s\n",
-                       node, len, len_local, (next!=END_TSO_QUEUE) ? "RBH" : ""));
-  */
+               fprintf(stderr,"## BQ Stats of %p: [%d entries] %s\n",
+                       node, len, (bqe!=END_BQ_QUEUE) ? "RBH" : ""));
 }
 #elif defined(PAR)
 void 
@@ -2001,6 +2451,134 @@ interruptStgRts(void)
    This has nothing to do with the UnblockThread event in GranSim. -- HWL
    -------------------------------------------------------------------------- */
 
+#if defined(GRAN) || defined(PAR)
+/*
+  NB: only the type of the blocking queue is different in GranSim and GUM
+      the operations on the queue-elements are the same
+      long live polymorphism!
+*/
+static void
+unblockThread(StgTSO *tso)
+{
+  StgBlockingQueueElement *t, **last;
+
+  ACQUIRE_LOCK(&sched_mutex);
+  switch (tso->why_blocked) {
+
+  case NotBlocked:
+    return;  /* not blocked */
+
+  case BlockedOnMVar:
+    ASSERT(get_itbl(tso->block_info.closure)->type == MVAR);
+    {
+      StgBlockingQueueElement *last_tso = END_BQ_QUEUE;
+      StgMVar *mvar = (StgMVar *)(tso->block_info.closure);
+
+      last = (StgBlockingQueueElement **)&mvar->head;
+      for (t = (StgBlockingQueueElement *)mvar->head; 
+          t != END_BQ_QUEUE; 
+          last = &t->link, last_tso = t, t = t->link) {
+       if (t == (StgBlockingQueueElement *)tso) {
+         *last = (StgBlockingQueueElement *)tso->link;
+         if (mvar->tail == tso) {
+           mvar->tail = (StgTSO *)last_tso;
+         }
+         goto done;
+       }
+      }
+      barf("unblockThread (MVAR): TSO not found");
+    }
+
+  case BlockedOnBlackHole:
+    ASSERT(get_itbl(tso->block_info.closure)->type == BLACKHOLE_BQ);
+    {
+      StgBlockingQueue *bq = (StgBlockingQueue *)(tso->block_info.closure);
+
+      last = &bq->blocking_queue;
+      for (t = bq->blocking_queue; 
+          t != END_BQ_QUEUE; 
+          last = &t->link, t = t->link) {
+       if (t == (StgBlockingQueueElement *)tso) {
+         *last = (StgBlockingQueueElement *)tso->link;
+         goto done;
+       }
+      }
+      barf("unblockThread (BLACKHOLE): TSO not found");
+    }
+
+  case BlockedOnException:
+    {
+      StgTSO *target  = tso->block_info.tso;
+
+      ASSERT(get_itbl(target)->type == TSO);
+      ASSERT(target->blocked_exceptions != NULL);
+
+      last = (StgBlockingQueueElement **)&target->blocked_exceptions;
+      for (t = (StgBlockingQueueElement *)target->blocked_exceptions; 
+          t != END_BQ_QUEUE; 
+          last = &t->link, t = t->link) {
+       ASSERT(get_itbl(t)->type == TSO);
+       if (t == (StgBlockingQueueElement *)tso) {
+         *last = (StgBlockingQueueElement *)tso->link;
+         goto done;
+       }
+      }
+      barf("unblockThread (Exception): TSO not found");
+    }
+
+  case BlockedOnRead:
+  case BlockedOnWrite:
+    {
+      StgBlockingQueueElement *prev = NULL;
+      for (t = (StgBlockingQueueElement *)blocked_queue_hd; t != END_BQ_QUEUE; 
+          prev = t, t = t->link) {
+       if (t == (StgBlockingQueueElement *)tso) {
+         if (prev == NULL) {
+           blocked_queue_hd = (StgTSO *)t->link;
+           if ((StgBlockingQueueElement *)blocked_queue_tl == t) {
+             blocked_queue_tl = END_TSO_QUEUE;
+           }
+         } else {
+           prev->link = t->link;
+           if ((StgBlockingQueueElement *)blocked_queue_tl == t) {
+             blocked_queue_tl = (StgTSO *)prev;
+           }
+         }
+         goto done;
+       }
+      }
+      barf("unblockThread (I/O): TSO not found");
+    }
+
+  case BlockedOnDelay:
+    {
+      StgBlockingQueueElement *prev = NULL;
+      for (t = (StgBlockingQueueElement *)sleeping_queue; t != END_BQ_QUEUE; 
+          prev = t, t = t->link) {
+       if (t == (StgBlockingQueueElement *)tso) {
+         if (prev == NULL) {
+           sleeping_queue = (StgTSO *)t->link;
+         } else {
+           prev->link = t->link;
+         }
+         goto done;
+       }
+      }
+      barf("unblockThread (I/O): TSO not found");
+    }
+
+  default:
+    barf("unblockThread");
+  }
+
+ done:
+  tso->link = END_TSO_QUEUE;
+  tso->why_blocked = NotBlocked;
+  tso->block_info.closure = NULL;
+  PUSH_ON_RUN_QUEUE(tso);
+  RELEASE_LOCK(&sched_mutex);
+}
+#else
 static void
 unblockThread(StgTSO *tso)
 {
@@ -2067,7 +2645,6 @@ unblockThread(StgTSO *tso)
       barf("unblockThread (Exception): TSO not found");
     }
 
-  case BlockedOnDelay:
   case BlockedOnRead:
   case BlockedOnWrite:
     {
@@ -2092,6 +2669,23 @@ unblockThread(StgTSO *tso)
       barf("unblockThread (I/O): TSO not found");
     }
 
+  case BlockedOnDelay:
+    {
+      StgTSO *prev = NULL;
+      for (t = sleeping_queue; t != END_TSO_QUEUE; 
+          prev = t, t = t->link) {
+       if (t == tso) {
+         if (prev == NULL) {
+           sleeping_queue = t->link;
+         } else {
+           prev->link = t->link;
+         }
+         goto done;
+       }
+      }
+      barf("unblockThread (I/O): TSO not found");
+    }
+
   default:
     barf("unblockThread");
   }
@@ -2103,6 +2697,7 @@ unblockThread(StgTSO *tso)
   PUSH_ON_RUN_QUEUE(tso);
   RELEASE_LOCK(&sched_mutex);
 }
+#endif
 
 /* -----------------------------------------------------------------------------
  * raiseAsync()
@@ -2149,7 +2744,7 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
   StgPtr          sp = tso->sp;
   
   /* Thread already dead? */
-  if (tso->whatNext == ThreadComplete || tso->whatNext == ThreadKilled) {
+  if (tso->what_next == ThreadComplete || tso->what_next == ThreadKilled) {
     return;
   }
 
@@ -2164,7 +2759,7 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
    * returns to the next return address on the stack.
    */
   if ( LOOKS_LIKE_GHC_INFO((void*)*sp) ) {
-    *(--sp) = (W_)&dummy_ret_closure;
+    *(--sp) = (W_)&stg_dummy_ret_closure;
   }
 
   while (1) {
@@ -2183,11 +2778,11 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
        */
       ap = (StgAP_UPD *)allocate(sizeofW(StgPAP) + 2);
       TICK_ALLOC_UPD_PAP(3,0);
-      SET_HDR(ap,&PAP_info,cf->header.prof.ccs);
+      SET_HDR(ap,&stg_PAP_info,cf->header.prof.ccs);
              
       ap->n_args = 2;
       ap->fun = cf->handler;   /* :: Exception -> IO a */
-      ap->payload[0] = (P_)exception;
+      ap->payload[0] = exception;
       ap->payload[1] = ARG_TAG(0); /* realworld token */
 
       /* throw away the stack from Sp up to and including the
@@ -2204,7 +2799,7 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
        * unblockAsyncExceptions_ret stack frame.
        */
       if (!cf->exceptions_blocked) {
-       *(sp--) = (W_)&unblockAsyncExceptionszh_ret_info;
+       *(sp--) = (W_)&stg_unblockAsyncExceptionszh_ret_info;
       }
       
       /* Ensure that async exceptions are blocked when running the handler.
@@ -2218,7 +2813,8 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
        */
       sp[0] = (W_)ap;
       tso->sp = sp;
-      tso->whatNext = ThreadEnterGHC;
+      tso->what_next = ThreadEnterGHC;
+      IF_DEBUG(sanity, checkTSO(tso));
       return;
     }
 
@@ -2234,14 +2830,14 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
     ap->fun    = (StgClosure *)sp[0];
     sp++;
     for(i=0; i < (nat)words; ++i) {
-      ap->payload[i] = (P_)*sp++;
+      ap->payload[i] = (StgClosure *)*sp++;
     }
     
     switch (get_itbl(su)->type) {
       
     case UPDATE_FRAME:
       {
-       SET_HDR(ap,&AP_UPD_info,su->header.prof.ccs /* ToDo */); 
+       SET_HDR(ap,&stg_AP_UPD_info,su->header.prof.ccs /* ToDo */); 
        TICK_ALLOC_UP_THK(words+1,0);
        
        IF_DEBUG(scheduler,
@@ -2270,13 +2866,13 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
        /* We want a PAP, not an AP_UPD.  Fortunately, the
         * layout's the same.
         */
-       SET_HDR(ap,&PAP_info,su->header.prof.ccs /* ToDo */);
+       SET_HDR(ap,&stg_PAP_info,su->header.prof.ccs /* ToDo */);
        TICK_ALLOC_UPD_PAP(words+1,0);
        
        /* now build o = FUN(catch,ap,handler) */
        o = (StgClosure *)allocate(sizeofW(StgClosure)+2);
        TICK_ALLOC_FUN(2,0);
-       SET_HDR(o,&catch_info,su->header.prof.ccs /* ToDo */);
+       SET_HDR(o,&stg_catch_info,su->header.prof.ccs /* ToDo */);
        o->payload[0] = (StgClosure *)ap;
        o->payload[1] = cf->handler;
        
@@ -2297,14 +2893,14 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
        StgSeqFrame *sf = (StgSeqFrame *)su;
        StgClosure* o;
        
-       SET_HDR(ap,&PAP_info,su->header.prof.ccs /* ToDo */);
+       SET_HDR(ap,&stg_PAP_info,su->header.prof.ccs /* ToDo */);
        TICK_ALLOC_UPD_PAP(words+1,0);
        
        /* now build o = FUN(seq,ap) */
        o = (StgClosure *)allocate(sizeofW(StgClosure)+1);
        TICK_ALLOC_SE_THK(1,0);
-       SET_HDR(o,&seq_info,su->header.prof.ccs /* ToDo */);
-       payloadCPtr(o,0) = (StgClosure *)ap;
+       SET_HDR(o,&stg_seq_info,su->header.prof.ccs /* ToDo */);
+       o->payload[0] = (StgClosure *)ap;
        
        IF_DEBUG(scheduler,
                 fprintf(stderr,  "scheduler: Built ");
@@ -2322,11 +2918,11 @@ raiseAsync(StgTSO *tso, StgClosure *exception)
       /* We've stripped the entire stack, the thread is now dead. */
       sp += sizeofW(StgStopFrame) - 1;
       sp[0] = (W_)exception;   /* save the exception */
-      tso->whatNext = ThreadKilled;
+      tso->what_next = ThreadKilled;
       tso->su = (StgUpdateFrame *)(sp+1);
       tso->sp = sp;
       return;
-      
+
     default:
       barf("raiseAsync");
     }
@@ -2373,6 +2969,61 @@ resurrectThreads( StgTSO *threads )
   }
 }
 
+/* -----------------------------------------------------------------------------
+ * Blackhole detection: if we reach a deadlock, test whether any
+ * threads are blocked on themselves.  Any threads which are found to
+ * be self-blocked get sent a NonTermination exception.
+ *
+ * This is only done in a deadlock situation in order to avoid
+ * performance overhead in the normal case.
+ * -------------------------------------------------------------------------- */
+
+static void
+detectBlackHoles( void )
+{
+    StgTSO *t = all_threads;
+    StgUpdateFrame *frame;
+    StgClosure *blocked_on;
+
+    for (t = all_threads; t != END_TSO_QUEUE; t = t->global_link) {
+
+       if (t->why_blocked != BlockedOnBlackHole) {
+           continue;
+       }
+
+       blocked_on = t->block_info.closure;
+
+       for (frame = t->su; ; frame = frame->link) {
+           switch (get_itbl(frame)->type) {
+
+           case UPDATE_FRAME:
+               if (frame->updatee == blocked_on) {
+                   /* We are blocking on one of our own computations, so
+                    * send this thread the NonTermination exception.  
+                    */
+                   IF_DEBUG(scheduler, 
+                            sched_belch("thread %d is blocked on itself", t->id));
+                   raiseAsync(t, (StgClosure *)NonTermination_closure);
+                   goto done;
+               }
+               else {
+                   continue;
+               }
+
+           case CATCH_FRAME:
+           case SEQ_FRAME:
+               continue;
+               
+           case STOP_FRAME:
+               break;
+           }
+           break;
+       }
+
+    done: ;
+    }   
+}
+
 //@node Debugging Routines, Index, Exception Handling Routines, Main scheduling code
 //@subsection Debugging Routines
 
@@ -2387,39 +3038,47 @@ printThreadBlockage(StgTSO *tso)
 {
   switch (tso->why_blocked) {
   case BlockedOnRead:
-    fprintf(stderr,"blocked on read from fd %d", tso->block_info.fd);
+    fprintf(stderr,"is blocked on read from fd %d", tso->block_info.fd);
     break;
   case BlockedOnWrite:
-    fprintf(stderr,"blocked on write to fd %d", tso->block_info.fd);
+    fprintf(stderr,"is blocked on write to fd %d", tso->block_info.fd);
     break;
   case BlockedOnDelay:
-    fprintf(stderr,"blocked on delay of %d ms", tso->block_info.delay);
+    fprintf(stderr,"is blocked until %d", tso->block_info.target);
     break;
   case BlockedOnMVar:
-    fprintf(stderr,"blocked on an MVar");
+    fprintf(stderr,"is blocked on an MVar");
     break;
   case BlockedOnException:
-    fprintf(stderr,"blocked on delivering an exception to thread %d",
+    fprintf(stderr,"is blocked on delivering an exception to thread %d",
            tso->block_info.tso->id);
     break;
   case BlockedOnBlackHole:
-    fprintf(stderr,"blocked on a black hole");
+    fprintf(stderr,"is blocked on a black hole");
     break;
   case NotBlocked:
-    fprintf(stderr,"not blocked");
+    fprintf(stderr,"is not blocked");
     break;
 #if defined(PAR)
   case BlockedOnGA:
-    fprintf(stderr,"blocked on global address");
+    fprintf(stderr,"is blocked on global address; local FM_BQ is %p (%s)",
+           tso->block_info.closure, info_type(tso->block_info.closure));
+    break;
+  case BlockedOnGA_NoSend:
+    fprintf(stderr,"is blocked on global address (no send); local FM_BQ is %p (%s)",
+           tso->block_info.closure, info_type(tso->block_info.closure));
     break;
 #endif
+  default:
+    barf("printThreadBlockage: strange tso->why_blocked: %d for TSO %d (%d)",
+        tso->why_blocked, tso->id, tso);
   }
 }
 
 void
 printThreadStatus(StgTSO *tso)
 {
-  switch (tso->whatNext) {
+  switch (tso->what_next) {
   case ThreadKilled:
     fprintf(stderr,"has been killed");
     break;
@@ -2438,7 +3097,7 @@ printAllThreads(void)
 
   sched_belch("all threads:");
   for (t = all_threads; t != END_TSO_QUEUE; t = t->global_link) {
-    fprintf(stderr, "\tthread %d is ", t->id);
+    fprintf(stderr, "\tthread %d ", t->id);
     printThreadStatus(t);
     fprintf(stderr,"\n");
   }
@@ -2494,9 +3153,9 @@ print_bq (StgClosure *node)
       break;
     case CONSTR:
       fprintf(stderr," %s (IP %p),",
-             (get_itbl(bqe) == &RBH_Save_0_info ? "RBH_Save_0" :
-              get_itbl(bqe) == &RBH_Save_1_info ? "RBH_Save_1" :
-              get_itbl(bqe) == &RBH_Save_2_info ? "RBH_Save_2" :
+             (get_itbl(bqe) == &stg_RBH_Save_0_info ? "RBH_Save_0" :
+              get_itbl(bqe) == &stg_RBH_Save_1_info ? "RBH_Save_1" :
+              get_itbl(bqe) == &stg_RBH_Save_2_info ? "RBH_Save_2" :
               "RBH_Save_?"), get_itbl(bqe));
       break;
     default:
@@ -2512,7 +3171,6 @@ void
 print_bq (StgClosure *node)
 {
   StgBlockingQueueElement *bqe;
-  StgTSO *tso;
   PEs node_loc, tso_loc;
   rtsBool end;
 
@@ -2534,7 +3192,7 @@ print_bq (StgClosure *node)
        !end; // iterate until bqe points to a CONSTR
        end = (get_itbl(bqe)->type == CONSTR) || (bqe->link==END_BQ_QUEUE), bqe = end ? END_BQ_QUEUE : bqe->link) {
     ASSERT(bqe != END_BQ_QUEUE);             // sanity check
-    ASSERT(bqe != (StgTSO*)NULL);            // sanity check
+    ASSERT(bqe != (StgBlockingQueueElement *)NULL);  // sanity check
     /* types of closures that may appear in a blocking queue */
     ASSERT(get_itbl(bqe)->type == TSO ||           
           get_itbl(bqe)->type == CONSTR); 
@@ -2544,19 +3202,19 @@ print_bq (StgClosure *node)
     tso_loc = where_is((StgClosure *)bqe);
     switch (get_itbl(bqe)->type) {
     case TSO:
-      fprintf(stderr," TSO %d (%x) on [PE %d],",
-             ((StgTSO *)bqe)->id, ((StgTSO *)bqe), tso_loc);
+      fprintf(stderr," TSO %d (%p) on [PE %d],",
+             ((StgTSO *)bqe)->id, (StgTSO *)bqe, tso_loc);
       break;
     case CONSTR:
       fprintf(stderr," %s (IP %p),",
-             (get_itbl(bqe) == &RBH_Save_0_info ? "RBH_Save_0" :
-              get_itbl(bqe) == &RBH_Save_1_info ? "RBH_Save_1" :
-              get_itbl(bqe) == &RBH_Save_2_info ? "RBH_Save_2" :
+             (get_itbl(bqe) == &stg_RBH_Save_0_info ? "RBH_Save_0" :
+              get_itbl(bqe) == &stg_RBH_Save_1_info ? "RBH_Save_1" :
+              get_itbl(bqe) == &stg_RBH_Save_2_info ? "RBH_Save_2" :
               "RBH_Save_?"), get_itbl(bqe));
       break;
     default:
       barf("Unexpected closure type %s in blocking queue of %p (%s)",
-          info_type(bqe), node, info_type(node));
+          info_type((StgClosure *)bqe), node, info_type(node));
       break;
     }
   } /* for */
@@ -2615,6 +3273,7 @@ sched_belch(char *s, ...)
 
 #endif /* DEBUG */
 
+
 //@node Index,  , Debugging Routines, Main scheduling code
 //@subsection Index