diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 42660486e..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "files.associations": { - "zmalloc.h": "c", - "stat.h": "c", - "array": "cpp", - "atomic": "cpp", - "*.tcc": "cpp", - "cctype": "cpp", - "chrono": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "condition_variable": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdint": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "deque": "cpp", - "list": "cpp", - "unordered_map": "cpp", - "vector": "cpp", - "exception": "cpp", - "fstream": "cpp", - "functional": "cpp", - "future": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "memory": "cpp", - "mutex": "cpp", - "new": "cpp", - "numeric": "cpp", - "optional": "cpp", - "ostream": "cpp", - "ratio": "cpp", - "scoped_allocator": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "streambuf": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "thread": "cpp", - "cinttypes": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "typeinfo": "cpp", - "utility": "cpp", - "set": "cpp", - "algorithm": "cpp" - } -} diff --git a/src/fastlock.cpp b/src/fastlock.cpp index 49c0fb095..861bfb7ca 100644 --- a/src/fastlock.cpp +++ b/src/fastlock.cpp @@ -43,6 +43,7 @@ #include #include #include +#include "config.h" #ifdef __APPLE__ #include @@ -60,6 +61,11 @@ #define UNUSED(x) ((void)x) #endif +#ifdef HAVE_BACKTRACE +#include +__attribute__((weak)) void logStackTrace(ucontext_t *) {} +#endif + extern int g_fInCrash; /**************************************************** @@ -149,6 +155,43 @@ __attribute__((weak)) void serverLog(int , const char *fmt, ...) printf("\n"); } +extern "C" pid_t gettid() +{ + static thread_local int pidCache = -1; +#ifdef __linux__ + if (pidCache == -1) + pidCache = syscall(SYS_gettid); +#else + if (pidCache == -1) { + uint64_t tidT; + pthread_threadid_np(nullptr, &tidT); + assert(tidT < UINT_MAX); + pidCache = (int)tidT; + } +#endif + return pidCache; +} + +void printTrace() +{ +#ifdef HAVE_BACKTRACE + serverLog(3 /*LL_WARNING*/, "printing backtrace for thread %d", gettid()); + ucontext_t ctxt; + getcontext(&ctxt); + logStackTrace(&ctxt); +#endif +} + + +#ifdef __linux__ +static int futex(volatile unsigned *uaddr, int futex_op, int val, + const struct timespec *timeout, int val3) +{ + return syscall(SYS_futex, uaddr, futex_op, val, + timeout, uaddr, val3); +} +#endif + class DeadlockDetector { std::map m_mapwait; @@ -156,9 +199,19 @@ class DeadlockDetector public: void registerwait(fastlock *lock, pid_t thispid) { + static volatile bool fInDeadlock = false; + if (lock == &m_lock || g_fInCrash) return; fastlock_lock(&m_lock); + + if (fInDeadlock) + { + printTrace(); + fastlock_unlock(&m_lock); + return; + } + m_mapwait.insert(std::make_pair(thispid, lock)); // Detect cycles @@ -184,6 +237,19 @@ public: if (pidCheck == thispid) break; } + // Wake All sleeping threads so they can print their callstacks +#ifdef HAVE_BACKTRACE +#ifdef __linux__ + int mask = -1; + fInDeadlock = true; + fastlock_unlock(&m_lock); + futex(&lock->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask); + futex(&itr->second->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask); + sleep(2); + fastlock_lock(&m_lock); + printTrace(); +#endif +#endif serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!"); _serverPanic(__FILE__, __LINE__, "Deadlock detected"); } @@ -222,32 +288,6 @@ uint64_t fastlock_getlongwaitcount() return rval; } -#ifdef __linux__ -static int futex(volatile unsigned *uaddr, int futex_op, int val, - const struct timespec *timeout, int val3) -{ - return syscall(SYS_futex, uaddr, futex_op, val, - timeout, uaddr, val3); -} -#endif - -extern "C" pid_t gettid() -{ - static thread_local int pidCache = -1; -#ifdef __linux__ - if (pidCache == -1) - pidCache = syscall(SYS_gettid); -#else - if (pidCache == -1) { - uint64_t tidT; - pthread_threadid_np(nullptr, &tidT); - assert(tidT < UINT_MAX); - pidCache = (int)tidT; - } -#endif - return pidCache; -} - extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask) { #ifdef __linux__ diff --git a/src/networking.cpp b/src/networking.cpp index 58bd79bcc..94b202c17 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -1633,7 +1633,10 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { c->lock.lock(); ae.arm(c); if (c->flags & CLIENT_CLOSE_ASAP) - freeClient(c); + { + if (!freeClient(c)) + c->lock.unlock(); + } } } @@ -3104,12 +3107,23 @@ int processEventsWhileBlocked(int iel) { int iterations = 4; /* See the function top-comment. */ int count = 0; - client *c = serverTL->current_client; - if (c != nullptr) + std::vector vecclients; + listIter li; + listNode *ln; + listRewind(g_pserver->clients, &li); + + // All client locks must be acquired *after* the global lock is reacquired to prevent deadlocks + // so unlock here, and save them for reacquisition later + while ((ln = listNext(&li)) != nullptr) { - serverAssert(c->flags & CLIENT_PROTECTED); - c->lock.unlock(); + client *c = (client*)listNodeValue(ln); + if (c->lock.fOwnLock()) { + serverAssert(c->flags & CLIENT_PROTECTED); // If the client is not protected we have no gurantee they won't be free'd in the event loop + c->lock.unlock(); + vecclients.push_back(c); + } } + aeReleaseLock(); try @@ -3126,18 +3140,18 @@ int processEventsWhileBlocked(int iel) { { // Caller expects us to be locked so fix and rethrow AeLocker locker; - if (c != nullptr) - c->lock.lock(); - locker.arm(c); + locker.arm(nullptr); locker.release(); + for (client *c : vecclients) + c->lock.lock(); throw; } AeLocker locker; - if (c != nullptr) - c->lock.lock(); - locker.arm(c); + locker.arm(nullptr); locker.release(); + for (client *c : vecclients) + c->lock.lock(); return count; } diff --git a/src/server.cpp b/src/server.cpp index 3d7aa3a41..2a3054973 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -1039,6 +1039,7 @@ struct redisCommand redisCommandTable[] = { /* We use a private localtime implementation which is fork-safe. The logging * function of Redis may be called from other threads. */ extern "C" void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst); +extern "C" pid_t gettid(); /* Low level logging. To use only for very big messages, otherwise * serverLog() is to prefer. */ @@ -1076,8 +1077,8 @@ void serverLogRaw(int level, const char *msg) { } else { role_char = (listLength(g_pserver->masters) ? 'S':'M'); /* Slave or Master. */ } - fprintf(fp,"%d:%c %s %c %s\n", - (int)getpid(),role_char, buf,c[level],msg); + fprintf(fp,"%d:%d:%c %s %c %s\n", + (int)getpid(),(int)gettid(),role_char, buf,c[level],msg); } fflush(fp); diff --git a/src/version.h b/src/version.h index db32931f1..751337aaa 100644 --- a/src/version.h +++ b/src/version.h @@ -1,3 +1,3 @@ -#define KEYDB_REAL_VERSION "0.0.0" +#define KEYDB_REAL_VERSION "5.3.2" extern const char *KEYDB_SET_VERSION; // Unlike real version, this can be overriden by the config