gvsafronov/futriix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Print stack traces of other threads in the deadlock detector

Browse Source 
Former-commit-id: 6b0172b9bf124372f4c8688c91c380c6c1b607c7
This commit is contained in:
John Sully 2020-03-19 15:37:49 -04:00
parent 1bdccb3c42
commit 69806aaa7b
1 changed files with 66 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/fastlock.cpp
View File

@ -43,6 +43,7 @@
#include <string.h> #include <string.h>
#include <stdarg.h> #include <stdarg.h>
#include <stdio.h> #include <stdio.h>
#include "config.h"
#ifdef __APPLE__ #ifdef __APPLE__
#include <TargetConditionals.h> #include <TargetConditionals.h>
@ -60,6 +61,11 @@
#define UNUSED(x) ((void)x) #define UNUSED(x) ((void)x)
#endif #endif
#ifdef HAVE_BACKTRACE
#include <ucontext.h>
__attribute__((weak)) void logStackTrace(ucontext_t *) {}
#endif
extern int g_fInCrash; extern int g_fInCrash;
/**************************************************** /****************************************************
@ -149,6 +155,43 @@ __attribute__((weak)) void serverLog(int , const char *fmt, ...)
printf("\n"); printf("\n");
} }
extern "C" pid_t gettid()
{
static thread_local int pidCache = -1;
#ifdef __linux__
if (pidCache == -1)
pidCache = syscall(SYS_gettid);
#else
if (pidCache == -1) {
uint64_t tidT;
pthread_threadid_np(nullptr, &tidT);
assert(tidT < UINT_MAX);
pidCache = (int)tidT;
}
#endif
return pidCache;
}
void printTrace()
{
#ifdef HAVE_BACKTRACE
serverLog(3 /*LL_WARNING*/, "printing backtrace for thread %d", gettid());
ucontext_t ctxt;
getcontext(&ctxt);
logStackTrace(&ctxt);
#endif
}
#ifdef __linux__
static int futex(volatile unsigned *uaddr, int futex_op, int val,
const struct timespec *timeout, int val3)
{
return syscall(SYS_futex, uaddr, futex_op, val,
timeout, uaddr, val3);
}
#endif
class DeadlockDetector class DeadlockDetector
{ {
std::map<pid_t, fastlock *> m_mapwait; std::map<pid_t, fastlock *> m_mapwait;
@ -156,9 +199,19 @@ class DeadlockDetector
public: public:
void registerwait(fastlock *lock, pid_t thispid) void registerwait(fastlock *lock, pid_t thispid)
{ {
static volatile bool fInDeadlock = false;
if (lock == &m_lock || g_fInCrash) if (lock == &m_lock || g_fInCrash)
return; return;
fastlock_lock(&m_lock); fastlock_lock(&m_lock);
if (fInDeadlock)
{
printTrace();
fastlock_unlock(&m_lock);
return;
}
m_mapwait.insert(std::make_pair(thispid, lock)); m_mapwait.insert(std::make_pair(thispid, lock));
// Detect cycles // Detect cycles
@ -184,6 +237,19 @@ public:
if (pidCheck == thispid) if (pidCheck == thispid)
break; break;
} }
// Wake All sleeping threads so they can print their callstacks
#ifdef HAVE_BACKTRACE
#ifdef __linux__
int mask = -1;
fInDeadlock = true;
fastlock_unlock(&m_lock);
futex(&lock->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask);
futex(&itr->second->m_ticket.u, FUTEX_WAKE_BITSET_PRIVATE, INT_MAX, nullptr, mask);
sleep(2);
fastlock_lock(&m_lock);
printTrace();
#endif
#endif
serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!"); serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!");
_serverPanic(__FILE__, __LINE__, "Deadlock detected"); _serverPanic(__FILE__, __LINE__, "Deadlock detected");
} }
@ -222,32 +288,6 @@ uint64_t fastlock_getlongwaitcount()
return rval; return rval;
} }
#ifdef __linux__
static int futex(volatile unsigned *uaddr, int futex_op, int val,
const struct timespec *timeout, int val3)
{
return syscall(SYS_futex, uaddr, futex_op, val,
timeout, uaddr, val3);
}
#endif
extern "C" pid_t gettid()
{
static thread_local int pidCache = -1;
#ifdef __linux__
if (pidCache == -1)
pidCache = syscall(SYS_gettid);
#else
if (pidCache == -1) {
uint64_t tidT;
pthread_threadid_np(nullptr, &tidT);
assert(tidT < UINT_MAX);
pidCache = (int)tidT;
}
#endif
return pidCache;
}
extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask) extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask)
{ {
#ifdef __linux__ #ifdef __linux__