Auto tune lock for high CPU tension scenarios

Former-commit-id: 8edbae2e04538f82a146a6c2b459a6dfcacf99b2

src/fastlock.cpp

@@ -40,6 +40,7 @@
 #include <map>
 #ifdef __linux__
 #include <linux/futex.h>
+#include <sys/sysinfo.h>
 #endif
 #include <string.h>
 #include <stdarg.h>
@@ -69,6 +70,8 @@ __attribute__((weak)) void logStackTrace(ucontext_t *) {}
 #endif
 
 extern int g_fInCrash;
+extern int g_fTestMode;
+int g_fHighCpuPressure = false;
 
 /****************************************************
  *
@@ -332,6 +335,7 @@ extern "C" void fastlock_lock(struct fastlock *lock)
     unsigned mask = (1U << (myticket % 32));
     unsigned cloops = 0;
     ticket ticketT;
+    unsigned loopLimit = g_fHighCpuPressure ? 0x10000 : 0x100000;
 
     for (;;)
     {
@@ -344,7 +348,7 @@ extern "C" void fastlock_lock(struct fastlock *lock)
 #elif defined(__aarch64__)
         __asm__ __volatile__ ("yield");
 #endif
-        if ((++cloops % 0x100000) == 0)
+        if ((++cloops % loopLimit) == 0)
         {
             fastlock_sleep(lock, tid, ticketT.u, mask);
         }
@@ -461,3 +465,24 @@ void fastlock_lock_recursive(struct fastlock *lock, int nesting)
     fastlock_lock(lock);
     lock->m_depth = nesting;
 }
+
+void fastlock_auto_adjust_waits()
+{
+#ifdef __linux__
+    struct sysinfo sysinf;
+    auto fHighPressurePrev = g_fHighCpuPressure;
+    memset(&sysinf, 0, sizeof sysinf);
+    if (!sysinfo(&sysinf)) {
+        auto avgCoreLoad = sysinf.loads[0] / get_nprocs();
+        g_fHighCpuPressure = (avgCoreLoad > ((1 << SI_LOAD_SHIFT) * 0.9));
+        if (g_fHighCpuPressure)
+            serverLog(!fHighPressurePrev ?  3 /*LL_WARNING*/ : 1 /* LL_VERBOSE */, "NOTICE: Detuning locks due to high load per core: %.2f%%", avgCoreLoad / (double)(1 << SI_LOAD_SHIFT)*100.0);
+    }
+
+    if (!g_fHighCpuPressure && fHighPressurePrev) {
+        serverLog(3 /*LL_WARNING*/, "NOTICE: CPU pressure reduced");
+    }
+#else
+    g_fHighCpuPressure = g_fTestMode;
+#endif
+}

src/fastlock.h

@@ -15,6 +15,7 @@ void fastlock_unlock(struct fastlock *lock);
 void fastlock_free(struct fastlock *lock);
 int fastlock_unlock_recursive(struct fastlock *lock);
 void fastlock_lock_recursive(struct fastlock *lock, int nesting);
+void fastlock_auto_adjust_waits();
 
 uint64_t fastlock_getlongwaitcount();   // this is a global value
 

src/fastlock_x64.asm

@@ -3,6 +3,7 @@
 
 .extern gettid
 .extern fastlock_sleep
+.extern g_fHighCpuPressure
 
 #	This is the first use of assembly in this codebase, a valid question is WHY?
 #	The spinlock we implement here is performance critical, and simply put GCC
@@ -33,6 +34,13 @@ fastlock_lock:
 	cmp [rdi], esi          # Is the TID we got back the owner of the lock?
 	je .LLocked             # Don't spin in that case
 
+	mov r9d, 0x1000         #	1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
+	mov eax, [rip+g_fHighCpuPressure]
+	test eax, eax
+	jz .LNoTestMode
+	mov r9d, 0x10000
+.LNoTestMode:
+
 	xor eax, eax            # eliminate partial register dependency
 	inc eax                 # we want to add one
 	lock xadd [rdi+66], ax  # do the xadd, ax contains the value before the addition
@@ -45,8 +53,7 @@ fastlock_lock:
 	cmp dx, ax              # is our ticket up?
 	je .LLocked             # leave the loop
 	pause
-	add ecx, 0x1000         # Have we been waiting a long time? (oflow if we have)
+	add ecx, r9d            # Have we been waiting a long time? (oflow if we have)
-	                        #	1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
 	jnc .LLoop              # If so, give up our timeslice to someone who's doing real work
 	# Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop"
 	#	But the compiler doesn't know that we rarely hit this, and when we do we know the lock is

src/redis-benchmark.cpp

@@ -66,6 +66,8 @@ struct benchmarkThread;
 struct clusterNode;
 struct redisConfig;
 
+int g_fTestMode = false;
+
 static struct config {
     aeEventLoop *el;
     const char *hostip;

src/redis-cli.c

@@ -69,6 +69,8 @@
 redisContext *context;
 struct config config;
 
+int g_fTestMode = 0;
+
 /* User preferences. */
 static struct pref {
     int hints;

src/server.cpp

@@ -2110,6 +2110,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
         migrateCloseTimedoutSockets();
     }
 
+    /* Tune the fastlock to CPU load */
+    run_with_period(30000) {
+        fastlock_auto_adjust_waits();
+    }
+
     /* Resize tracking keys table if needed. This is also done at every
      * command execution, but we want to be sure that if the last command
      * executed changes the value via CONFIG SET, the server will perform