threadStackOverflow: check whether stack squeezing released some stack (#3677)

In a stack overflow situation, stack squeezing may reduce the stack
size, but we don't know whether it has been reduced enough for the
stack check to succeed if we try again.  Fortunately stack squeezing
is idempotent, so all we need to do is record whether *any* squeezing
happened.  If we are at the stack's absolute -K limit, and stack
squeezing happened, then we try running the thread again.

We also want to avoid enlarging the stack if squeezing has already
released some of it.  However, we don't want to get into a
pathalogical situation where a thread has a nearly full stack (near
its current limit, but not near the absolute -K limit), keeps
allocating a little bit, squeezing removes a little bit, and then it
runs again.  So to avoid this, if we squeezed *and* there is still
less than BLOCK_SIZE_W words free, then we enlarge the stack anyway.

diff --git a/includes/rts/Constants.h b/includes/rts/Constants.h
index 82f3bdd..8ebe16a 100644 (file)
--- a/includes/rts/Constants.h
+++ b/includes/rts/Constants.h
@@ -263,6 +263,13 @@
  */
 #define TSO_MARKED 64
 
+/*
+ * Used to communicate between stackSqueeze() and
+ * threadStackOverflow() that a thread's stack was squeezed and the
+ * stack may not need to be expanded.
+ */
+#define TSO_SQUEEZED 128
+
 /* -----------------------------------------------------------------------------
    RET_DYN stack frames
    -------------------------------------------------------------------------- */

diff --git a/rts/Schedule.c b/rts/Schedule.c
index 998d846..dbee436 100644 (file)
--- a/rts/Schedule.c
+++ b/rts/Schedule.c
@@ -2189,12 +2189,27 @@ threadStackOverflow(Capability *cap, StgTSO *tso)
   // while we are moving the TSO:
   lockClosure((StgClosure *)tso);
 
-  if (tso->stack_size >= tso->max_stack_size && !(tso->flags & TSO_BLOCKEX)) {
+  if (tso->stack_size >= tso->max_stack_size
+      && !(tso->flags & TSO_BLOCKEX)) {
       // NB. never raise a StackOverflow exception if the thread is
       // inside Control.Exceptino.block.  It is impractical to protect
       // against stack overflow exceptions, since virtually anything
       // can raise one (even 'catch'), so this is the only sensible
       // thing to do here.  See bug #767.
+      //
+
+      if (tso->flags & TSO_SQUEEZED) {
+          return tso;
+      }
+      // #3677: In a stack overflow situation, stack squeezing may
+      // reduce the stack size, but we don't know whether it has been
+      // reduced enough for the stack check to succeed if we try
+      // again.  Fortunately stack squeezing is idempotent, so all we
+      // need to do is record whether *any* squeezing happened.  If we
+      // are at the stack's absolute -K limit, and stack squeezing
+      // happened, then we try running the thread again.  The
+      // TSO_SQUEEZED flag is set by threadPaused() to tell us whether
+      // squeezing happened or not.
 
       debugTrace(DEBUG_gc,
                 "threadStackOverflow of TSO %ld (%p): stack too large (now %ld; max is %ld)",
@@ -2210,6 +2225,21 @@ threadStackOverflow(Capability *cap, StgTSO *tso)
       return tso;
   }
 
+
+  // We also want to avoid enlarging the stack if squeezing has
+  // already released some of it.  However, we don't want to get into
+  // a pathalogical situation where a thread has a nearly full stack
+  // (near its current limit, but not near the absolute -K limit),
+  // keeps allocating a little bit, squeezing removes a little bit,
+  // and then it runs again.  So to avoid this, if we squeezed *and*
+  // there is still less than BLOCK_SIZE_W words free, then we enlarge
+  // the stack anyway.
+  if ((tso->flags & TSO_SQUEEZED) && 
+      ((W_)(tso->sp - tso->stack) >= BLOCK_SIZE_W)) {
+      unlockTSO(tso);
+      return tso;
+  }
+
   /* Try to double the current stack size.  If that takes us over the
    * maximum stack size for this thread, then use the maximum instead
    * (that is, unless we're already at or over the max size and we

diff --git a/rts/ThreadPaused.c b/rts/ThreadPaused.c
index 58c30e3..96a2367 100644 (file)
--- a/rts/ThreadPaused.c
+++ b/rts/ThreadPaused.c
@@ -317,5 +317,10 @@ end:
     if (RtsFlags.GcFlags.squeezeUpdFrames == rtsTrue &&
        ((weight <= 5 && words_to_squeeze > 0) || weight < words_to_squeeze)) {
        stackSqueeze(tso, (StgPtr)frame);
+        tso->flags |= TSO_SQUEEZED;
+        // This flag tells threadStackOverflow() that the stack was
+        // squeezed, because it may not need to be expanded.
+    } else {
+        tso->flags &= ~TSO_SQUEEZED;
     }
 }