Merge branch 'RELEASE_5' into unstable

Former-commit-id: a0bc2084128f991697d3e1b7473d14c55afdd785

.vscode/settings.json

@@ -1,58 +0,0 @@
-{
-    "files.associations": {
-        "zmalloc.h": "c",
-        "stat.h": "c",
-        "array": "cpp",
-        "atomic": "cpp",
-        "*.tcc": "cpp",
-        "cctype": "cpp",
-        "chrono": "cpp",
-        "clocale": "cpp",
-        "cmath": "cpp",
-        "condition_variable": "cpp",
-        "cstdarg": "cpp",
-        "cstddef": "cpp",
-        "cstdint": "cpp",
-        "cstdio": "cpp",
-        "cstdlib": "cpp",
-        "cstring": "cpp",
-        "ctime": "cpp",
-        "cwchar": "cpp",
-        "cwctype": "cpp",
-        "deque": "cpp",
-        "list": "cpp",
-        "unordered_map": "cpp",
-        "vector": "cpp",
-        "exception": "cpp",
-        "fstream": "cpp",
-        "functional": "cpp",
-        "future": "cpp",
-        "initializer_list": "cpp",
-        "iomanip": "cpp",
-        "iosfwd": "cpp",
-        "iostream": "cpp",
-        "istream": "cpp",
-        "limits": "cpp",
-        "memory": "cpp",
-        "mutex": "cpp",
-        "new": "cpp",
-        "numeric": "cpp",
-        "optional": "cpp",
-        "ostream": "cpp",
-        "ratio": "cpp",
-        "scoped_allocator": "cpp",
-        "sstream": "cpp",
-        "stdexcept": "cpp",
-        "streambuf": "cpp",
-        "string_view": "cpp",
-        "system_error": "cpp",
-        "thread": "cpp",
-        "cinttypes": "cpp",
-        "tuple": "cpp",
-        "type_traits": "cpp",
-        "typeinfo": "cpp",
-        "utility": "cpp",
-        "set": "cpp",
-        "algorithm": "cpp"
-    }
-}

src/fastlock.cpp

@@ -43,6 +43,7 @@
 #include <string.h>
 #include <stdarg.h>
 #include <stdio.h>
+#include "config.h"
 
 #ifdef __APPLE__
 #include <TargetConditionals.h>
@@ -60,6 +61,11 @@
 #define UNUSED(x) ((void)x)
 #endif
 
+#ifdef HAVE_BACKTRACE
+#include <ucontext.h>
+__attribute__((weak)) void logStackTrace(ucontext_t *) {}
+#endif
+
 extern int g_fInCrash;
 
 /****************************************************
@@ -149,6 +155,43 @@ __attribute__((weak)) void serverLog(int , const char *fmt, ...)
     printf("\n");
 }
 
+extern "C" pid_t gettid()
+{
+    static thread_local int pidCache = -1;
+#ifdef __linux__
+    if (pidCache == -1)
+        pidCache = syscall(SYS_gettid);
+#else
+	if (pidCache == -1) {
+		uint64_t tidT;
+		pthread_threadid_np(nullptr, &tidT);
+		assert(tidT < UINT_MAX);
+		pidCache = (int)tidT;
+	}
+#endif
+    return pidCache;
+}
+
+void printTrace()
+{
+#ifdef HAVE_BACKTRACE
+    serverLog(3 /*LL_WARNING*/, "printing backtrace for thread %d", gettid());
+    ucontext_t ctxt;
+    getcontext(&ctxt);
+    logStackTrace(&ctxt);
+#endif
+}
+
+
+#ifdef __linux__
+static int futex(volatile unsigned *uaddr, int futex_op, int val,
+    const struct timespec *timeout, int val3)
+{
+    return syscall(SYS_futex, uaddr, futex_op, val,
+                    timeout, uaddr, val3);
+}
+#endif
+
 class DeadlockDetector
 {
     std::map<pid_t, fastlock *> m_mapwait;
@@ -156,9 +199,19 @@ class DeadlockDetector
 public:
     void registerwait(fastlock *lock, pid_t thispid)
     {
+        static volatile bool fInDeadlock = false;
+
         if (lock == &m_lock || g_fInCrash)
             return;
         fastlock_lock(&m_lock);
+        
+        if (fInDeadlock)
+        {
+            printTrace();
+            fastlock_unlock(&m_lock);
+            return;
+        }
+
         m_mapwait.insert(std::make_pair(thispid, lock));
 
         // Detect cycles
@@ -184,6 +237,19 @@ public:
                     if (pidCheck == thispid)
                         break;
                 }
+                // Wake All sleeping threads so they can print their callstacks
+#ifdef HAVE_BACKTRACE
+#ifdef __linux__
+                int mask = -1;
+                fInDeadlock = true;
+                fastlock_unlock(&m_lock);
+                futex(&lock->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask);
+                futex(&itr->second->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask);
+                sleep(2);
+                fastlock_lock(&m_lock);
+                printTrace();
+#endif
+#endif
                 serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!");
                 _serverPanic(__FILE__, __LINE__, "Deadlock detected");
             }
@@ -222,32 +288,6 @@ uint64_t fastlock_getlongwaitcount()
     return rval;
 }
 
-#ifdef __linux__
-static int futex(volatile unsigned *uaddr, int futex_op, int val,
-    const struct timespec *timeout, int val3)
-{
-    return syscall(SYS_futex, uaddr, futex_op, val,
-                    timeout, uaddr, val3);
-}
-#endif
-
-extern "C" pid_t gettid()
-{
-    static thread_local int pidCache = -1;
-#ifdef __linux__
-    if (pidCache == -1)
-        pidCache = syscall(SYS_gettid);
-#else
-	if (pidCache == -1) {
-		uint64_t tidT;
-		pthread_threadid_np(nullptr, &tidT);
-		assert(tidT < UINT_MAX);
-		pidCache = (int)tidT;
-	}
-#endif
-    return pidCache;
-}
-
 extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask)
 {
 #ifdef __linux__

src/networking.cpp

@@ -1633,7 +1633,10 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) {
         c->lock.lock();
         ae.arm(c);
         if (c->flags & CLIENT_CLOSE_ASAP)
-            freeClient(c);
+        {
+            if (!freeClient(c))
+                c->lock.unlock();
+        }
     }
 }
 
@@ -3104,13 +3107,24 @@ int processEventsWhileBlocked(int iel) {
     int iterations = 4; /* See the function top-comment. */
     int count = 0;
 
-    client *c = serverTL->current_client;
-    if (c != nullptr)
+    std::vector<client*> vecclients;
+    listIter li;
+    listNode *ln;
+    listRewind(g_pserver->clients, &li);
+
+    // All client locks must be acquired *after* the global lock is reacquired to prevent deadlocks
+    //  so unlock here, and save them for reacquisition later
+    while ((ln = listNext(&li)) != nullptr)
     {
-        serverAssert(c->flags & CLIENT_PROTECTED);
-        c->lock.unlock();
+        client *c = (client*)listNodeValue(ln);
+        if (c->lock.fOwnLock()) {
+            serverAssert(c->flags & CLIENT_PROTECTED);  // If the client is not protected we have no gurantee they won't be free'd in the event loop
+            c->lock.unlock();
+            vecclients.push_back(c);
+        }
     }
     
+
     aeReleaseLock();
     try
     {
@@ -3126,18 +3140,18 @@ int processEventsWhileBlocked(int iel) {
     {
         // Caller expects us to be locked so fix and rethrow
         AeLocker locker;
-        if (c != nullptr)
-            c->lock.lock();
-        locker.arm(c);
+        locker.arm(nullptr);
         locker.release();
+        for (client *c : vecclients)
+            c->lock.lock();
         throw;
     }
     
     AeLocker locker;
-    if (c != nullptr)
-        c->lock.lock();
-    locker.arm(c);
+    locker.arm(nullptr);
     locker.release();
+    for (client *c : vecclients)
+        c->lock.lock();
     return count;
 }
 

src/server.cpp

@@ -1039,6 +1039,7 @@ struct redisCommand redisCommandTable[] = {
 /* We use a private localtime implementation which is fork-safe. The logging
  * function of Redis may be called from other threads. */
 extern "C" void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst);
+extern "C" pid_t gettid();
 
 /* Low level logging. To use only for very big messages, otherwise
  * serverLog() is to prefer. */
@@ -1076,8 +1077,8 @@ void serverLogRaw(int level, const char *msg) {
         } else {
             role_char = (listLength(g_pserver->masters) ? 'S':'M'); /* Slave or Master. */
         }
-        fprintf(fp,"%d:%c %s %c %s\n",
-            (int)getpid(),role_char, buf,c[level],msg);
+        fprintf(fp,"%d:%d:%c %s %c %s\n",
+            (int)getpid(),(int)gettid(),role_char, buf,c[level],msg);
     }
     fflush(fp);
 

src/version.h

@@ -1,3 +1,3 @@
-#define KEYDB_REAL_VERSION "0.0.0"
+#define KEYDB_REAL_VERSION "5.3.2"
 extern const char *KEYDB_SET_VERSION;   // Unlike real version, this can be overriden by the config