Optimize deadlock detection, fix callstack for ASM, and annotate locks
Note: This change moves our assembly code to use the GNU Assembler because NASM seems to be incapable of emitting the necessary debug information for callstack unwinding to work. Former-commit-id: 600fc241cfe79b9b32ac6010c6ea0c66747f0f15
This commit is contained in:
parent
b84d9671ec
commit
1c1260d71f
@ -190,7 +190,7 @@ endif
|
||||
|
||||
REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
|
||||
REDIS_CXX=$(QUIET_CC)$(CC) $(FINAL_CXXFLAGS)
|
||||
REDIS_NASM=$(QUIET_CC)nasm -felf64
|
||||
KEYDB_AS=$(QUIET_CC) as --64 -g
|
||||
REDIS_LD=$(QUIET_LINK)$(CXX) $(FINAL_LDFLAGS)
|
||||
REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL)
|
||||
|
||||
@ -295,7 +295,7 @@ dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c
|
||||
$(REDIS_CXX) -c $<
|
||||
|
||||
%.o: %.asm .make-prerequisites
|
||||
$(REDIS_NASM) $<
|
||||
$(KEYDB_AS) $< -o $@
|
||||
|
||||
clean:
|
||||
rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark
|
||||
|
@ -80,7 +80,7 @@ public:
|
||||
mutex_wrapper g_lock;
|
||||
|
||||
#else
|
||||
fastlock g_lock;
|
||||
fastlock g_lock("AE (global)");
|
||||
#endif
|
||||
thread_local aeEventLoop *g_eventLoopThisThread = NULL;
|
||||
|
||||
@ -327,7 +327,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
|
||||
for (i = 0; i < setsize; i++)
|
||||
eventLoop->events[i].mask = AE_NONE;
|
||||
|
||||
fastlock_init(&eventLoop->flock);
|
||||
fastlock_init(&eventLoop->flock, "event loop");
|
||||
int rgfd[2];
|
||||
if (pipe(rgfd) < 0)
|
||||
goto err;
|
||||
|
@ -678,7 +678,7 @@ client *createFakeClient(void) {
|
||||
c->puser = NULL;
|
||||
listSetFreeMethod(c->reply,freeClientReplyValue);
|
||||
listSetDupMethod(c->reply,dupClientReplyValue);
|
||||
fastlock_init(&c->lock);
|
||||
fastlock_init(&c->lock, "fake client");
|
||||
fastlock_lock(&c->lock);
|
||||
initClientMultiState(c);
|
||||
return c;
|
||||
|
@ -41,6 +41,7 @@
|
||||
#include <linux/futex.h>
|
||||
#endif
|
||||
#include <string.h>
|
||||
#include <stdarg.h>
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <TargetConditionals.h>
|
||||
@ -127,15 +128,25 @@
|
||||
#endif
|
||||
|
||||
#pragma weak _serverPanic
|
||||
extern "C" void _serverPanic(const char * /*file*/, int /*line*/, const char * /*msg*/, ...)
|
||||
extern "C" __attribute__((weak)) void _serverPanic(const char * /*file*/, int /*line*/, const char * /*msg*/, ...)
|
||||
{
|
||||
*((char*)-1) = 'x';
|
||||
}
|
||||
|
||||
#pragma weak serverLog
|
||||
__attribute__((weak)) void serverLog(int , const char *fmt, ...)
|
||||
{
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vprintf(fmt, args);
|
||||
va_end(args);
|
||||
printf("\n");
|
||||
}
|
||||
|
||||
class DeadlockDetector
|
||||
{
|
||||
std::map<pid_t, fastlock *> m_mapwait;
|
||||
fastlock m_lock;
|
||||
fastlock m_lock { "deadlock detector" };
|
||||
public:
|
||||
void registerwait(fastlock *lock, pid_t thispid)
|
||||
{
|
||||
@ -146,6 +157,7 @@ public:
|
||||
|
||||
// Detect cycles
|
||||
pid_t pidCheck = thispid;
|
||||
size_t cchecks = 0;
|
||||
for (;;)
|
||||
{
|
||||
auto itr = m_mapwait.find(pidCheck);
|
||||
@ -153,7 +165,26 @@ public:
|
||||
break;
|
||||
pidCheck = itr->second->m_pidOwner;
|
||||
if (pidCheck == thispid)
|
||||
{
|
||||
// Deadlock detected, printout some debugging info and crash
|
||||
serverLog(3 /*LL_WARNING*/, "\n\n");
|
||||
serverLog(3 /*LL_WARNING*/, "!!! ERROR: Deadlock detected !!!");
|
||||
pidCheck = thispid;
|
||||
for (;;)
|
||||
{
|
||||
auto itr = m_mapwait.find(pidCheck);
|
||||
serverLog(3 /* LL_WARNING */, "\t%d: (%p) %s", pidCheck, itr->second, itr->second->szName);
|
||||
pidCheck = itr->second->m_pidOwner;
|
||||
if (pidCheck == thispid)
|
||||
break;
|
||||
}
|
||||
serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!");
|
||||
_serverPanic(__FILE__, __LINE__, "Deadlock detected");
|
||||
}
|
||||
|
||||
if (cchecks > m_mapwait.size())
|
||||
break; // There is a cycle but we're not in it
|
||||
++cchecks;
|
||||
}
|
||||
fastlock_unlock(&m_lock);
|
||||
}
|
||||
@ -170,16 +201,6 @@ public:
|
||||
|
||||
DeadlockDetector g_dlock;
|
||||
|
||||
extern "C" void registerwait(fastlock *lock, pid_t thispid)
|
||||
{
|
||||
g_dlock.registerwait(lock, thispid);
|
||||
}
|
||||
|
||||
extern "C" void clearwait(fastlock *lock, pid_t thispid)
|
||||
{
|
||||
g_dlock.clearwait(lock, thispid);
|
||||
}
|
||||
|
||||
static_assert(sizeof(pid_t) <= sizeof(fastlock::m_pidOwner), "fastlock::m_pidOwner not large enough");
|
||||
uint64_t g_longwaits = 0;
|
||||
|
||||
@ -190,7 +211,6 @@ uint64_t fastlock_getlongwaitcount()
|
||||
return rval;
|
||||
}
|
||||
|
||||
#ifndef ASM_SPINLOCK
|
||||
#ifdef __linux__
|
||||
static int futex(volatile unsigned *uaddr, int futex_op, int val,
|
||||
const struct timespec *timeout, int val3)
|
||||
@ -199,7 +219,6 @@ static int futex(volatile unsigned *uaddr, int futex_op, int val,
|
||||
timeout, uaddr, val3);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
extern "C" pid_t gettid()
|
||||
{
|
||||
@ -218,13 +237,26 @@ extern "C" pid_t gettid()
|
||||
return pidCache;
|
||||
}
|
||||
|
||||
extern "C" void fastlock_init(struct fastlock *lock)
|
||||
extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask)
|
||||
{
|
||||
#ifdef __linux__
|
||||
g_dlock.registerwait(lock, pid);
|
||||
__atomic_fetch_or(&lock->futex, mask, __ATOMIC_ACQUIRE);
|
||||
futex(&lock->m_ticket.u, FUTEX_WAIT_BITSET_PRIVATE, wake, nullptr, mask);
|
||||
__atomic_fetch_and(&lock->futex, ~mask, __ATOMIC_RELEASE);
|
||||
g_dlock.clearwait(lock, pid);
|
||||
#endif
|
||||
__atomic_fetch_add(&g_longwaits, 1, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
extern "C" void fastlock_init(struct fastlock *lock, const char *name)
|
||||
{
|
||||
lock->m_ticket.m_active = 0;
|
||||
lock->m_ticket.m_avail = 0;
|
||||
lock->m_depth = 0;
|
||||
lock->m_pidOwner = -1;
|
||||
lock->futex = 0;
|
||||
lock->szName = name;
|
||||
ANNOTATE_RWLOCK_CREATE(lock);
|
||||
}
|
||||
|
||||
@ -241,36 +273,23 @@ extern "C" void fastlock_lock(struct fastlock *lock)
|
||||
|
||||
int tid = gettid();
|
||||
unsigned myticket = __atomic_fetch_add(&lock->m_ticket.m_avail, 1, __ATOMIC_RELEASE);
|
||||
#ifdef __linux__
|
||||
unsigned mask = (1U << (myticket % 32));
|
||||
#endif
|
||||
int cloops = 0;
|
||||
ticket ticketT;
|
||||
|
||||
__atomic_load(&lock->m_ticket.u, &ticketT.u, __ATOMIC_ACQUIRE);
|
||||
if ((ticketT.u & 0xffff) != myticket)
|
||||
for (;;)
|
||||
{
|
||||
registerwait(lock, tid);
|
||||
for (;;)
|
||||
{
|
||||
__atomic_load(&lock->m_ticket.u, &ticketT.u, __ATOMIC_ACQUIRE);
|
||||
if ((ticketT.u & 0xffff) == myticket)
|
||||
break;
|
||||
__atomic_load(&lock->m_ticket.u, &ticketT.u, __ATOMIC_ACQUIRE);
|
||||
if ((ticketT.u & 0xffff) == myticket)
|
||||
break;
|
||||
|
||||
#if defined(__i386__) || defined(__amd64__)
|
||||
__asm__ ("pause");
|
||||
__asm__ ("pause");
|
||||
#endif
|
||||
if ((++cloops % 1024*1024) == 0)
|
||||
{
|
||||
#ifdef __linux__
|
||||
__atomic_fetch_or(&lock->futex, mask, __ATOMIC_ACQUIRE);
|
||||
futex(&lock->m_ticket.u, FUTEX_WAIT_BITSET_PRIVATE, ticketT.u, nullptr, mask);
|
||||
__atomic_fetch_and(&lock->futex, ~mask, __ATOMIC_RELEASE);
|
||||
#endif
|
||||
__atomic_fetch_add(&g_longwaits, 1, __ATOMIC_RELAXED);
|
||||
}
|
||||
if ((++cloops % 1024*1024) == 0)
|
||||
{
|
||||
fastlock_sleep(lock, tid, ticketT.u, mask);
|
||||
}
|
||||
clearwait(lock, tid);
|
||||
}
|
||||
|
||||
lock->m_depth = 1;
|
||||
|
@ -7,7 +7,7 @@ extern "C" {
|
||||
|
||||
/* Begin C API */
|
||||
struct fastlock;
|
||||
void fastlock_init(struct fastlock *lock);
|
||||
void fastlock_init(struct fastlock *lock, const char *name);
|
||||
void fastlock_lock(struct fastlock *lock);
|
||||
int fastlock_trylock(struct fastlock *lock, int fWeak);
|
||||
void fastlock_unlock(struct fastlock *lock);
|
||||
@ -45,24 +45,25 @@ struct fastlock
|
||||
volatile int m_pidOwner;
|
||||
volatile int m_depth;
|
||||
unsigned futex;
|
||||
const char *szName;
|
||||
|
||||
#ifdef __cplusplus
|
||||
fastlock()
|
||||
fastlock(const char *name)
|
||||
{
|
||||
fastlock_init(this);
|
||||
fastlock_init(this, name);
|
||||
}
|
||||
|
||||
void lock()
|
||||
inline void lock()
|
||||
{
|
||||
fastlock_lock(this);
|
||||
}
|
||||
|
||||
bool try_lock(bool fWeak = false)
|
||||
inline bool try_lock(bool fWeak = false)
|
||||
{
|
||||
return !!fastlock_trylock(this, fWeak);
|
||||
}
|
||||
|
||||
void unlock()
|
||||
inline void unlock()
|
||||
{
|
||||
fastlock_unlock(this);
|
||||
}
|
||||
|
@ -1,178 +1,160 @@
|
||||
section .text
|
||||
.intel_syntax noprefix
|
||||
.text
|
||||
|
||||
extern gettid
|
||||
extern sched_yield
|
||||
extern g_longwaits
|
||||
extern registerwait
|
||||
extern clearwait
|
||||
.extern gettid
|
||||
.extern fastlock_sleep
|
||||
|
||||
; This is the first use of assembly in this codebase, a valid question is WHY?
|
||||
; The spinlock we implement here is performance critical, and simply put GCC
|
||||
; emits awful code. The original C code is left in fastlock.cpp for reference
|
||||
; and x-plat.
|
||||
# This is the first use of assembly in this codebase, a valid question is WHY?
|
||||
# The spinlock we implement here is performance critical, and simply put GCC
|
||||
# emits awful code. The original C code is left in fastlock.cpp for reference
|
||||
# and x-plat.
|
||||
|
||||
ALIGN 16
|
||||
global fastlock_lock
|
||||
.ALIGN 16
|
||||
.global fastlock_lock
|
||||
.type fastlock_lock,@function
|
||||
fastlock_lock:
|
||||
; RDI points to the struct:
|
||||
; uint16_t active
|
||||
; uint16_t avail
|
||||
; int32_t m_pidOwner
|
||||
; int32_t m_depth
|
||||
.cfi_startproc
|
||||
.cfi_def_cfa rsp, 8
|
||||
# RDI points to the struct:
|
||||
# uint16_t active
|
||||
# uint16_t avail
|
||||
# int32_t m_pidOwner
|
||||
# int32_t m_depth
|
||||
|
||||
; First get our TID and put it in ecx
|
||||
push rdi ; we need our struct pointer (also balance the stack for the call)
|
||||
call gettid ; get our thread ID (TLS is nasty in ASM so don't bother inlining)
|
||||
mov esi, eax ; back it up in esi
|
||||
pop rdi ; get our pointer back
|
||||
# First get our TID and put it in ecx
|
||||
push rdi # we need our struct pointer (also balance the stack for the call)
|
||||
.cfi_adjust_cfa_offset 8
|
||||
call gettid # get our thread ID (TLS is nasty in ASM so don't bother inlining)
|
||||
mov esi, eax # back it up in esi
|
||||
pop rdi # get our pointer back
|
||||
.cfi_adjust_cfa_offset -8
|
||||
|
||||
cmp [rdi+4], esi ; Is the TID we got back the owner of the lock?
|
||||
je .LLocked ; Don't spin in that case
|
||||
cmp [rdi+4], esi # Is the TID we got back the owner of the lock?
|
||||
je .LLocked # Don't spin in that case
|
||||
|
||||
xor eax, eax ; eliminate partial register dependency
|
||||
inc eax ; we want to add one
|
||||
lock xadd [rdi+2], ax ; do the xadd, ax contains the value before the addition
|
||||
; ax now contains the ticket
|
||||
mov edx, [rdi]
|
||||
cmp dx, ax ; is our ticket up?
|
||||
je .LLocked ; no need to loop
|
||||
; Lock is contended, so inform the deadlock detector
|
||||
push rax
|
||||
push rdi
|
||||
push rsi
|
||||
call registerwait
|
||||
pop rsi
|
||||
pop rdi
|
||||
pop rax
|
||||
; OK Start the wait loop
|
||||
xor eax, eax # eliminate partial register dependency
|
||||
inc eax # we want to add one
|
||||
lock xadd [rdi+2], ax # do the xadd, ax contains the value before the addition
|
||||
# ax now contains the ticket
|
||||
# OK Start the wait loop
|
||||
xor ecx, ecx
|
||||
ALIGN 16
|
||||
.ALIGN 16
|
||||
.LLoop:
|
||||
mov edx, [rdi]
|
||||
cmp dx, ax ; is our ticket up?
|
||||
je .LExitLoop ; leave the loop
|
||||
cmp dx, ax # is our ticket up?
|
||||
je .LLocked # leave the loop
|
||||
pause
|
||||
add ecx, 1000h ; Have we been waiting a long time? (oflow if we have)
|
||||
; 1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
|
||||
jnc .LLoop ; If so, give up our timeslice to someone who's doing real work
|
||||
; Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop"
|
||||
; But the compiler doesn't know that we rarely hit this, and when we do we know the lock is
|
||||
; taking a long time to be released anyways. We optimize for the common case of short
|
||||
; lock intervals. That's why we're using a spinlock in the first place
|
||||
; If we get here we're going to sleep in the kernel with a futex
|
||||
add ecx, 0x1000 # Have we been waiting a long time? (oflow if we have)
|
||||
# 1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
|
||||
jnc .LLoop # If so, give up our timeslice to someone who's doing real work
|
||||
# Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop"
|
||||
# But the compiler doesn't know that we rarely hit this, and when we do we know the lock is
|
||||
# taking a long time to be released anyways. We optimize for the common case of short
|
||||
# lock intervals. That's why we're using a spinlock in the first place
|
||||
# If we get here we're going to sleep in the kernel with a futex
|
||||
push rdi
|
||||
push rsi
|
||||
push rax
|
||||
; Setup the syscall args
|
||||
; rdi ARG1 futex (already in rdi)
|
||||
mov esi, (9 | 128) ; rsi ARG2 FUTEX_WAIT_BITSET_PRIVATE
|
||||
; rdx ARG3 ticketT.u (already in edx)
|
||||
xor r10d, r10d ; r10 ARG4 NULL
|
||||
mov r8, rdi ; r8 ARG5 dup rdi
|
||||
xor r9d, r9d
|
||||
bts r9d, eax ; r9 ARG6 mask
|
||||
mov eax, 202 ; sys_futex
|
||||
; Do the syscall
|
||||
lock or [rdi+12], r9d ; inform the unlocking thread we're waiting
|
||||
syscall ; wait for the futex
|
||||
not r9d ; convert our flag into a mask of bits not to touch
|
||||
lock and [rdi+12], r9d ; clear the flag in the futex control mask
|
||||
; cleanup and continue
|
||||
mov rcx, g_longwaits
|
||||
inc qword [rcx] ; increment our long wait counter
|
||||
.cfi_adjust_cfa_offset 24
|
||||
# Setup the syscall args
|
||||
|
||||
# rdi ARG1 futex (already in rdi)
|
||||
# rsi ARG2 tid (already in esi)
|
||||
# rdx ARG3 ticketT.u (already in edx)
|
||||
bts ecx, eax # rcx ARG4 mask
|
||||
call fastlock_sleep
|
||||
# cleanup and continue
|
||||
pop rax
|
||||
pop rsi
|
||||
xor ecx, ecx ; Reset our loop counter
|
||||
jmp .LLoop ; Get back in the game
|
||||
ALIGN 16
|
||||
.LExitLoop:
|
||||
push rsi
|
||||
push rdi
|
||||
call clearwait
|
||||
pop rdi
|
||||
pop rsi
|
||||
ALIGN 16
|
||||
.cfi_adjust_cfa_offset -24
|
||||
xor ecx, ecx # Reset our loop counter
|
||||
jmp .LLoop # Get back in the game
|
||||
.ALIGN 16
|
||||
.LLocked:
|
||||
mov [rdi+4], esi ; lock->m_pidOwner = gettid()
|
||||
inc dword [rdi+8] ; lock->m_depth++
|
||||
mov [rdi+4], esi # lock->m_pidOwner = gettid()
|
||||
inc dword ptr [rdi+8] # lock->m_depth++
|
||||
ret
|
||||
.cfi_endproc
|
||||
|
||||
ALIGN 16
|
||||
global fastlock_trylock
|
||||
.ALIGN 16
|
||||
.global fastlock_trylock
|
||||
.type fastlock_trylock,@function
|
||||
fastlock_trylock:
|
||||
; RDI points to the struct:
|
||||
; uint16_t active
|
||||
; uint16_t avail
|
||||
; int32_t m_pidOwner
|
||||
; int32_t m_depth
|
||||
# RDI points to the struct:
|
||||
# uint16_t active
|
||||
# uint16_t avail
|
||||
# int32_t m_pidOwner
|
||||
# int32_t m_depth
|
||||
|
||||
; First get our TID and put it in ecx
|
||||
push rdi ; we need our struct pointer (also balance the stack for the call)
|
||||
call gettid ; get our thread ID (TLS is nasty in ASM so don't bother inlining)
|
||||
mov esi, eax ; back it up in esi
|
||||
pop rdi ; get our pointer back
|
||||
# First get our TID and put it in ecx
|
||||
push rdi # we need our struct pointer (also balance the stack for the call)
|
||||
call gettid # get our thread ID (TLS is nasty in ASM so don't bother inlining)
|
||||
mov esi, eax # back it up in esi
|
||||
pop rdi # get our pointer back
|
||||
|
||||
cmp [rdi+4], esi ; Is the TID we got back the owner of the lock?
|
||||
je .LRecursive ; Don't spin in that case
|
||||
cmp [rdi+4], esi # Is the TID we got back the owner of the lock?
|
||||
je .LRecursive # Don't spin in that case
|
||||
|
||||
mov eax, [rdi] ; get both active and avail counters
|
||||
mov ecx, eax ; duplicate in ecx
|
||||
ror ecx, 16 ; swap upper and lower 16-bits
|
||||
cmp eax, ecx ; are the upper and lower 16-bits the same?
|
||||
jnz .LAlreadyLocked ; If not return failure
|
||||
mov eax, [rdi] # get both active and avail counters
|
||||
mov ecx, eax # duplicate in ecx
|
||||
ror ecx, 16 # swap upper and lower 16-bits
|
||||
cmp eax, ecx # are the upper and lower 16-bits the same?
|
||||
jnz .LAlreadyLocked # If not return failure
|
||||
|
||||
; at this point we know eax+ecx have [avail][active] and they are both the same
|
||||
add ecx, 10000h ; increment avail, ecx is now our wanted value
|
||||
lock cmpxchg [rdi], ecx ; If rdi still contains the value in eax, put in ecx (inc avail)
|
||||
jnz .LAlreadyLocked ; If Z is not set then someone locked it while we were preparing
|
||||
# at this point we know eax+ecx have [avail][active] and they are both the same
|
||||
add ecx, 0x10000 # increment avail, ecx is now our wanted value
|
||||
lock cmpxchg [rdi], ecx # If rdi still contains the value in eax, put in ecx (inc avail)
|
||||
jnz .LAlreadyLocked # If Z is not set then someone locked it while we were preparing
|
||||
xor eax, eax
|
||||
inc eax ; return SUCCESS! (eax=1)
|
||||
mov [rdi+4], esi ; lock->m_pidOwner = gettid()
|
||||
mov dword [rdi+8], eax ; lock->m_depth = 1
|
||||
inc eax # return SUCCESS! (eax=1)
|
||||
mov [rdi+4], esi # lock->m_pidOwner = gettid()
|
||||
mov dword ptr [rdi+8], eax # lock->m_depth = 1
|
||||
ret
|
||||
ALIGN 16
|
||||
.ALIGN 16
|
||||
.LRecursive:
|
||||
xor eax, eax
|
||||
inc eax ; return SUCCESS! (eax=1)
|
||||
inc dword [rdi+8] ; lock->m_depth++
|
||||
inc eax # return SUCCESS! (eax=1)
|
||||
inc dword ptr [rdi+8] # lock->m_depth++
|
||||
ret
|
||||
ALIGN 16
|
||||
.ALIGN 16
|
||||
.LAlreadyLocked:
|
||||
xor eax, eax ; return 0;
|
||||
xor eax, eax # return 0
|
||||
ret
|
||||
|
||||
ALIGN 16
|
||||
global fastlock_unlock
|
||||
.ALIGN 16
|
||||
.global fastlock_unlock
|
||||
fastlock_unlock:
|
||||
; RDI points to the struct:
|
||||
; uint16_t active
|
||||
; uint16_t avail
|
||||
; int32_t m_pidOwner
|
||||
; int32_t m_depth
|
||||
# RDI points to the struct:
|
||||
# uint16_t active
|
||||
# uint16_t avail
|
||||
# int32_t m_pidOwner
|
||||
# int32_t m_depth
|
||||
push r11
|
||||
sub dword [rdi+8], 1 ; decrement m_depth, don't use dec because it partially writes the flag register and we don't know its state
|
||||
jnz .LDone ; if depth is non-zero this is a recursive unlock, and we still hold it
|
||||
mov dword [rdi+4], -1 ; pidOwner = -1 (we don't own it anymore)
|
||||
mov ecx, [rdi] ; get current active (this one)
|
||||
inc ecx ; bump it to the next thread
|
||||
mov [rdi], cx ; give up our ticket (note: lock is not required here because the spinlock itself guards this variable)
|
||||
; At this point the lock is removed, however we must wake up any pending futexs
|
||||
mov r9d, 1 ; eax is the bitmask for 2 threads
|
||||
rol r9d, cl ; place the mask in the right spot for the next 2 threads
|
||||
ALIGN 16
|
||||
sub dword ptr [rdi+8], 1 # decrement m_depth, don't use dec because it partially writes the flag register and we don't know its state
|
||||
jnz .LDone # if depth is non-zero this is a recursive unlock, and we still hold it
|
||||
mov dword ptr [rdi+4], -1 # pidOwner = -1 (we don't own it anymore)
|
||||
mov ecx, [rdi] # get current active (this one)
|
||||
inc ecx # bump it to the next thread
|
||||
mov [rdi], cx # give up our ticket (note: lock is not required here because the spinlock itself guards this variable)
|
||||
# At this point the lock is removed, however we must wake up any pending futexs
|
||||
mov r9d, 1 # eax is the bitmask for 2 threads
|
||||
rol r9d, cl # place the mask in the right spot for the next 2 threads
|
||||
.ALIGN 16
|
||||
.LRetryWake:
|
||||
mov r11d, [rdi+12] ; load the futex mask
|
||||
and r11d, r9d ; are any threads waiting on a futex?
|
||||
jz .LDone ; if not we're done.
|
||||
; we have to wake the futexs
|
||||
; rdi ARG1 futex (already in rdi)
|
||||
mov esi, (10 | 128) ; rsi ARG2 FUTEX_WAKE_BITSET_PRIVATE
|
||||
mov edx, 0x7fffffff ; rdx ARG3 INT_MAX (number of threads to wake)
|
||||
xor r10d, r10d ; r10 ARG4 NULL
|
||||
mov r8, rdi ; r8 ARG5 dup rdi
|
||||
; r9 ARG6 mask (already set above)
|
||||
mov eax, 202 ; sys_futex
|
||||
mov r11d, [rdi+12] # load the futex mask
|
||||
and r11d, r9d # are any threads waiting on a futex?
|
||||
jz .LDone # if not we're done.
|
||||
# we have to wake the futexs
|
||||
# rdi ARG1 futex (already in rdi)
|
||||
mov esi, (10 | 128) # rsi ARG2 FUTEX_WAKE_BITSET_PRIVATE
|
||||
mov edx, 0x7fffffff # rdx ARG3 INT_MAX (number of threads to wake)
|
||||
xor r10d, r10d # r10 ARG4 NULL
|
||||
mov r8, rdi # r8 ARG5 dup rdi
|
||||
# r9 ARG6 mask (already set above)
|
||||
mov eax, 202 # sys_futex
|
||||
syscall
|
||||
cmp eax, 1 ; did we wake as many as we expected?
|
||||
cmp eax, 1 # did we wake as many as we expected?
|
||||
jnz .LRetryWake
|
||||
.LDone:
|
||||
pop r11
|
||||
|
@ -116,7 +116,7 @@ client *createClient(int fd, int iel) {
|
||||
uint64_t client_id;
|
||||
client_id = g_pserver->next_client_id.fetch_add(1);
|
||||
c->iel = iel;
|
||||
fastlock_init(&c->lock);
|
||||
fastlock_init(&c->lock, "client");
|
||||
c->id = client_id;
|
||||
c->resp = 2;
|
||||
c->fd = fd;
|
||||
|
@ -2878,7 +2878,7 @@ static void initServerThread(struct redisServerThreadVars *pvar, int fMain)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
fastlock_init(&pvar->lockPendingWrite);
|
||||
fastlock_init(&pvar->lockPendingWrite, "lockPendingWrite");
|
||||
|
||||
if (!fMain)
|
||||
{
|
||||
@ -2925,8 +2925,6 @@ void initServer(void) {
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
setupSignalHandlers();
|
||||
|
||||
fastlock_init(&g_pserver->flock);
|
||||
|
||||
g_pserver->db = (redisDb*)zmalloc(sizeof(redisDb)*cserver.dbnum, MALLOC_LOCAL);
|
||||
|
||||
/* Create the Redis databases, and initialize other internal state. */
|
||||
|
@ -1044,7 +1044,7 @@ typedef struct clientReplyBlock {
|
||||
* database. The database number is the 'id' field in the structure. */
|
||||
typedef struct redisDb {
|
||||
redisDb()
|
||||
: expireitr(nullptr)
|
||||
: expireitr(nullptr), lock("redisDB")
|
||||
{};
|
||||
dict *pdict; /* The keyspace for this DB */
|
||||
expireset *setexpire;
|
||||
@ -1437,7 +1437,7 @@ struct redisServerThreadVars {
|
||||
client blocked on a module command needs
|
||||
to be processed. */
|
||||
client *lua_client = nullptr; /* The "fake client" to query Redis from Lua */
|
||||
struct fastlock lockPendingWrite;
|
||||
struct fastlock lockPendingWrite { "thread pending write" };
|
||||
char neterr[ANET_ERR_LEN]; /* Error buffer for anet.c */
|
||||
long unsigned commandsExecuted = 0;
|
||||
};
|
||||
@ -1819,8 +1819,6 @@ struct redisServer {
|
||||
|
||||
int fActiveReplica; /* Can this replica also be a master? */
|
||||
|
||||
struct fastlock flock;
|
||||
|
||||
// Format:
|
||||
// Lower 20 bits: a counter incrementing for each command executed in the same millisecond
|
||||
// Upper 44 bits: mstime (least significant 44-bits) enough for ~500 years before rollover from date of addition
|
||||
|
Loading…
x
Reference in New Issue
Block a user