From 65ac2f4cefcea7ca78a65ca22889b51b5a27d1f0 Mon Sep 17 00:00:00 2001
From: Simon Marlow <marlowsd@gmail.com>
Date: Fri, 22 Jan 2010 16:49:11 +0000
Subject: [PATCH] When acquiring a spinlock, yieldThread() every 1000 spins
 (#3553, #3758)

This helps when the thread holding the lock has been descheduled,
which is the main cause of the "last-core slowdown" problem.  With
this patch, I get much better results with -N8 on an 8-core box,
although some benchmarks are still worse than with 7 cores.

I also added a yieldThread() into the any_work() loop of the parallel
GC when it has no work to do. Oddly, this seems to improve performance
on the parallel GC benchmarks even when all the cores are busy.
Perhaps it is due to reducing contention on the memory bus.
---
 includes/rts/Constants.h             |    9 +++++++++
 includes/rts/SpinLock.h              |   29 ++++++++++++++++++-----------
 includes/rts/storage/SMPClosureOps.h |    2 --
 rts/sm/GC.c                          |    3 +++
 4 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/includes/rts/Constants.h b/includes/rts/Constants.h
index 0aee60a..54a1ca7 100644
--- a/includes/rts/Constants.h
+++ b/includes/rts/Constants.h
@@ -296,4 +296,13 @@
 #error RESERVED_STACK_WORDS may be wrong!
 #endif
 
+/*
+ * The number of times we spin in a spin lock before yielding (see
+ * #3758).  To tune this value, use the benchmark in #3758: run the
+ * server with -N2 and the client both on a dual-core.  Also make sure
+ * that the chosen value doesn't slow down any of the parallel
+ * benchmarks in nofib/parallel.
+ */
+#define SPIN_COUNT 1000
+
 #endif /* RTS_CONSTANTS_H */
diff --git a/includes/rts/SpinLock.h b/includes/rts/SpinLock.h
index 3d0b56c..8b337de 100644
--- a/includes/rts/SpinLock.h
+++ b/includes/rts/SpinLock.h
@@ -36,7 +36,6 @@ typedef StgWord SpinLock;
 
 typedef lnat SpinLockCount;
 
-
 #if defined(PROF_SPIN)
 
 // PROF_SPIN enables counting the number of times we spin on a lock
@@ -45,13 +44,16 @@ typedef lnat SpinLockCount;
 INLINE_HEADER void ACQUIRE_SPIN_LOCK(SpinLock * p)
 {
     StgWord32 r = 0;
-spin:
-    r = cas((StgVolatilePtr)&(p->lock), 1, 0);
-    if (r == 0) {
-        p->spin++;
-        busy_wait_nop();
-        goto spin;
-    }
+    nat i;
+    do {
+        for (i = 0; i < SPIN_COUNT; i++) {
+            r = cas((StgVolatilePtr)&(p->lock), 1, 0);
+            if (r != 0) return;
+            p->spin++;
+            busy_wait_nop();
+        }
+        yieldThread();
+    } while (1);
 }
 
 // release spin lock
@@ -75,10 +77,15 @@ INLINE_HEADER void initSpinLock(SpinLock * p)
 INLINE_HEADER void ACQUIRE_SPIN_LOCK(SpinLock * p)
 {
     StgWord32 r = 0;
+    nat i;
     do {
-        r = cas((StgVolatilePtr)p, 1, 0);
-        busy_wait_nop();
-    } while(r == 0);
+        for (i = 0; i < SPIN_COUNT; i++) {
+            r = cas((StgVolatilePtr)p, 1, 0);
+            if (r != 0) return;
+            busy_wait_nop();
+        }
+        yieldThread();
+    } while (1);
 }
 
 // release spin lock
diff --git a/includes/rts/storage/SMPClosureOps.h b/includes/rts/storage/SMPClosureOps.h
index d5f7c3f..582ec0e 100644
--- a/includes/rts/storage/SMPClosureOps.h
+++ b/includes/rts/storage/SMPClosureOps.h
@@ -28,8 +28,6 @@ EXTERN_INLINE void unlockClosure(StgClosure *p, const StgInfoTable *info);
  * This is used primarily in the implementation of MVars.
  * -------------------------------------------------------------------------- */
 
-#define SPIN_COUNT 4000
-
 // We want a callable copy of lockClosure() so that we can refer to it
 // from .cmm files compiled using the native codegen.
 EXTERN_INLINE StgInfoTable *lockClosure(StgClosure *p)
diff --git a/rts/sm/GC.c b/rts/sm/GC.c
index 3bd5017..2eabdab 100644
--- a/rts/sm/GC.c
+++ b/rts/sm/GC.c
@@ -997,6 +997,9 @@ any_work (void)
 #endif
 
     gct->no_work++;
+#if defined(THREADED_RTS)
+    yieldThread();
+#endif
 
     return rtsFalse;
 }    
-- 
1.7.10.4