diff --git a/src/fastlock.cpp b/src/fastlock.cpp index 28092828f..d8fea0b6c 100644 --- a/src/fastlock.cpp +++ b/src/fastlock.cpp @@ -40,6 +40,7 @@ #include #ifdef __linux__ #include +#include #endif #include #include @@ -69,6 +70,8 @@ __attribute__((weak)) void logStackTrace(ucontext_t *) {} #endif extern int g_fInCrash; +extern int g_fTestMode; +int g_fHighCpuPressure = false; /**************************************************** * @@ -332,6 +335,7 @@ extern "C" void fastlock_lock(struct fastlock *lock) unsigned mask = (1U << (myticket % 32)); unsigned cloops = 0; ticket ticketT; + unsigned loopLimit = g_fHighCpuPressure ? 0x10000 : 0x100000; for (;;) { @@ -344,7 +348,7 @@ extern "C" void fastlock_lock(struct fastlock *lock) #elif defined(__aarch64__) __asm__ __volatile__ ("yield"); #endif - if ((++cloops % 0x100000) == 0) + if ((++cloops % loopLimit) == 0) { fastlock_sleep(lock, tid, ticketT.u, mask); } @@ -461,3 +465,24 @@ void fastlock_lock_recursive(struct fastlock *lock, int nesting) fastlock_lock(lock); lock->m_depth = nesting; } + +void fastlock_auto_adjust_waits() +{ +#ifdef __linux__ + struct sysinfo sysinf; + auto fHighPressurePrev = g_fHighCpuPressure; + memset(&sysinf, 0, sizeof sysinf); + if (!sysinfo(&sysinf)) { + auto avgCoreLoad = sysinf.loads[0] / get_nprocs(); + g_fHighCpuPressure = (avgCoreLoad > ((1 << SI_LOAD_SHIFT) * 0.9)); + if (g_fHighCpuPressure) + serverLog(!fHighPressurePrev ? 3 /*LL_WARNING*/ : 1 /* LL_VERBOSE */, "NOTICE: Detuning locks due to high load per core: %.2f%%", avgCoreLoad / (double)(1 << SI_LOAD_SHIFT)*100.0); + } + + if (!g_fHighCpuPressure && fHighPressurePrev) { + serverLog(3 /*LL_WARNING*/, "NOTICE: CPU pressure reduced"); + } +#else + g_fHighCpuPressure = g_fTestMode; +#endif +} \ No newline at end of file diff --git a/src/fastlock.h b/src/fastlock.h index cdf8fe454..e4ab1874f 100644 --- a/src/fastlock.h +++ b/src/fastlock.h @@ -15,6 +15,7 @@ void fastlock_unlock(struct fastlock *lock); void fastlock_free(struct fastlock *lock); int fastlock_unlock_recursive(struct fastlock *lock); void fastlock_lock_recursive(struct fastlock *lock, int nesting); +void fastlock_auto_adjust_waits(); uint64_t fastlock_getlongwaitcount(); // this is a global value diff --git a/src/fastlock_x64.asm b/src/fastlock_x64.asm index bcea0e095..35a11f2ba 100644 --- a/src/fastlock_x64.asm +++ b/src/fastlock_x64.asm @@ -3,6 +3,7 @@ .extern gettid .extern fastlock_sleep +.extern g_fHighCpuPressure # This is the first use of assembly in this codebase, a valid question is WHY? # The spinlock we implement here is performance critical, and simply put GCC @@ -33,6 +34,13 @@ fastlock_lock: cmp [rdi], esi # Is the TID we got back the owner of the lock? je .LLocked # Don't spin in that case + mov r9d, 0x1000 # 1000h is set so we overflow on the 1024*1024'th iteration (like the C code) + mov eax, [rip+g_fHighCpuPressure] + test eax, eax + jz .LNoTestMode + mov r9d, 0x10000 +.LNoTestMode: + xor eax, eax # eliminate partial register dependency inc eax # we want to add one lock xadd [rdi+66], ax # do the xadd, ax contains the value before the addition @@ -45,8 +53,7 @@ fastlock_lock: cmp dx, ax # is our ticket up? je .LLocked # leave the loop pause - add ecx, 0x1000 # Have we been waiting a long time? (oflow if we have) - # 1000h is set so we overflow on the 1024*1024'th iteration (like the C code) + add ecx, r9d # Have we been waiting a long time? (oflow if we have) jnc .LLoop # If so, give up our timeslice to someone who's doing real work # Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop" # But the compiler doesn't know that we rarely hit this, and when we do we know the lock is diff --git a/src/redis-benchmark.cpp b/src/redis-benchmark.cpp index 5ee6b8dba..e3935cae6 100644 --- a/src/redis-benchmark.cpp +++ b/src/redis-benchmark.cpp @@ -66,6 +66,8 @@ struct benchmarkThread; struct clusterNode; struct redisConfig; +int g_fTestMode = false; + static struct config { aeEventLoop *el; const char *hostip; diff --git a/src/redis-cli.c b/src/redis-cli.c index 142608a3a..d818fa4e9 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -69,6 +69,8 @@ redisContext *context; struct config config; +int g_fTestMode = 0; + /* User preferences. */ static struct pref { int hints; diff --git a/src/server.cpp b/src/server.cpp index 92bff8cd7..3cd5a8cee 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2110,6 +2110,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { migrateCloseTimedoutSockets(); } + /* Tune the fastlock to CPU load */ + run_with_period(30000) { + fastlock_auto_adjust_waits(); + } + /* Resize tracking keys table if needed. This is also done at every * command execution, but we want to be sure that if the last command * executed changes the value via CONFIG SET, the server will perform