Fix async safety in signal handlers (#12658)
see discussion from after https://github.com/redis/redis/pull/12453 was merged ---- This PR replaces signals that are not considered async-signal-safe (AS-safe) with safe calls. #### **1. serverLog() and serverLogFromHandler()** `serverLog` uses unsafe calls. It was decided that we will **avoid** `serverLog` calls by the signal handlers when: * The signal is not fatal, such as SIGALRM. In these cases, we prefer using `serverLogFromHandler` which is the safe version of `serverLog`. Note they have different prompts: `serverLog`: `62220:M 26 Oct 2023 14:39:04.526 # <msg>` `serverLogFromHandler`: `62220:signal-handler (1698331136) <msg>` * The code was added recently. Calls to `serverLog` by the signal handler have been there ever since Redis exists and it hasn't caused problems so far. To avoid regression, from now we should use `serverLogFromHandler` #### **2. `snprintf` `fgets` and `strtoul`(base = 16) --------> `_safe_snprintf`, `fgets_async_signal_safe`, `string_to_hex`** The safe version of `snprintf` was taken from [here](8cfc4ca5e7/src/mc_util.c (L754)) #### **3. fopen(), fgets(), fclose() --------> open(), read(), close()** #### **4. opendir(), readdir(), closedir() --------> open(), syscall(SYS_getdents64), close()** #### **5. Threads_mngr sync mechanisms** * waiting for the thread to generate stack trace: semaphore --------> busy-wait * `globals_rw_lock` was removed: as we are not using malloc and the semaphore anymore we don't need to protect `ThreadsManager_cleanups`. #### **6. Stacktraces buffer** The initial problem was that we were not able to safely call malloc within the signal handler. To solve that we created a buffer on the stack of `writeStacktraces` and saved it in a global pointer, assuming that under normal circumstances, the function `writeStacktraces` would complete before any thread attempted to write to it. However, **if threads lag behind, they might access this global pointer after it no longer belongs to the `writeStacktraces` stack, potentially corrupting memory.** To address this, various solutions were discussed [here](https://github.com/redis/redis/pull/12658#discussion_r1390442896) Eventually, we decided to **create a pipe** at server startup that will remain valid as long as the process is alive. We chose this solution due to its minimal memory usage, and since `write()` and `read()` are atomic operations. It ensures that stack traces from different threads won't mix. **The stacktraces collection process is now as follows:** * Cleaning the pipe to eliminate writes of late threads from previous runs. * Each thread writes to the pipe its stacktrace * Waiting for all the threads to mark completion or until a timeout (2 sec) is reached * Reading from the pipe to print the stacktraces. #### **7. Changes that were considered and eventually were dropped** * replace watchdog timer with a POSIX timer: according to [settimer man](https://linux.die.net/man/2/setitimer) > POSIX.1-2008 marks getitimer() and setitimer() obsolete, recommending the use of the POSIX timers API ([timer_gettime](https://linux.die.net/man/2/timer_gettime)(2), [timer_settime](https://linux.die.net/man/2/timer_settime)(2), etc.) instead. However, although it is supposed to conform to POSIX std, POSIX timers API is not supported on Mac. You can take a look here at the Linux implementation: [here](c7562ee135) To avoid messing up the code, and uncertainty regarding compatibility, it was decided to drop it for now. * avoid using sds (uses malloc) in logConfigDebugInfo It was considered to print config info instead of using sds, however apparently, `logConfigDebugInfo` does more than just print the sds, so it was decided this fix is out of this issue scope. #### **8. fix Signal mask check** The check `signum & sig_mask` intended to indicate whether the signal is blocked by the thread was incorrect. Actually, the bit position in the signal mask corresponds to the signal number. We fixed this by changing the condition to: `sig_mask & (1L << (sig_num - 1))` #### **9. Unrelated changes** both `fork.tcl `and `util.tcl` implemented a function called `count_log_message` expecting different parameters. This caused confusion when trying to run daily tests with additional test parameters to run a specific test. The `count_log_message` in `fork.tcl` was removed and the calls were replaced with calls to `count_log_message` located in `util.tcl` --------- Co-authored-by: Ozan Tezcan <ozantezcan@gmail.com> Co-authored-by: Oran Agra <oran@redislabs.com>
This commit is contained in:
meiravgri
GitHub
parent
5e403099bd
commit
2e854bccc6
228
src/debug.c
228
src/debug.c
@ -1029,7 +1029,7 @@ NULL
|
||||
|
||||
/* =========================== Crash handling ============================== */
|
||||
|
||||
__attribute__ ((noinline))
|
||||
__attribute__ ((noinline))
|
||||
void _serverAssert(const char *estr, const char *file, int line) {
|
||||
bugReportStart();
|
||||
serverLog(LL_WARNING,"=== ASSERTION FAILED ===");
|
||||
@ -1119,7 +1119,7 @@ void _serverAssertWithInfo(const client *c, const robj *o, const char *estr, con
|
||||
_serverAssert(estr,file,line);
|
||||
}
|
||||
|
||||
__attribute__ ((noinline))
|
||||
__attribute__ ((noinline))
|
||||
void _serverPanic(const char *file, int line, const char *msg, ...) {
|
||||
va_list ap;
|
||||
va_start(ap,msg);
|
||||
@ -1796,18 +1796,31 @@ void closeDirectLogFiledes(int fd) {
|
||||
if (!log_to_stdout) close(fd);
|
||||
}
|
||||
|
||||
#if defined(HAVE_BACKTRACE) && defined(__linux__)
|
||||
static int stacktrace_pipe[2] = {0};
|
||||
static void setupStacktracePipe(void) {
|
||||
if (-1 == anetPipe(stacktrace_pipe, O_CLOEXEC | O_NONBLOCK, O_CLOEXEC | O_NONBLOCK)) {
|
||||
serverLog(LL_WARNING, "setupStacktracePipe failed: %s", strerror(errno));
|
||||
}
|
||||
}
|
||||
#else
|
||||
static void setupStacktracePipe(void) {/* we don't need a pipe to write the stacktraces */}
|
||||
#endif
|
||||
#ifdef HAVE_BACKTRACE
|
||||
#define BACKTRACE_MAX_SIZE 100
|
||||
|
||||
#ifdef __linux__
|
||||
#if !defined(_GNU_SOURCE)
|
||||
#define _GNU_SOURCE
|
||||
#endif
|
||||
#include <sys/prctl.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <dirent.h>
|
||||
|
||||
#define TIDS_MAX_SIZE 50
|
||||
static size_t get_ready_to_signal_threads_tids(pid_t pid, int sig_num, pid_t tids[TIDS_MAX_SIZE]);
|
||||
static size_t get_ready_to_signal_threads_tids(int sig_num, pid_t tids[TIDS_MAX_SIZE]);
|
||||
|
||||
#define MAX_BUFF_LENGTH 256
|
||||
typedef struct {
|
||||
char thread_name[16];
|
||||
int trace_size;
|
||||
@ -1815,64 +1828,47 @@ typedef struct {
|
||||
void *trace[BACKTRACE_MAX_SIZE];
|
||||
} stacktrace_data;
|
||||
|
||||
static stacktrace_data *stacktraces_mempool = NULL;
|
||||
static redisAtomic size_t g_thread_ids = 0;
|
||||
|
||||
static stacktrace_data *get_stack_trace_pool(void) {
|
||||
size_t thread_id;
|
||||
atomicGetIncr(g_thread_ids, thread_id, 1);
|
||||
return stacktraces_mempool + thread_id;
|
||||
}
|
||||
|
||||
__attribute__ ((noinline)) static void collect_stacktrace_data(void) {
|
||||
/* allocate stacktrace_data struct */
|
||||
stacktrace_data *trace_data = get_stack_trace_pool();
|
||||
stacktrace_data trace_data = {{0}};
|
||||
|
||||
/* Get the stack trace first! */
|
||||
trace_data->trace_size = backtrace(trace_data->trace, BACKTRACE_MAX_SIZE);
|
||||
trace_data.trace_size = backtrace(trace_data.trace, BACKTRACE_MAX_SIZE);
|
||||
|
||||
/* get the thread name */
|
||||
prctl(PR_GET_NAME, trace_data->thread_name);
|
||||
prctl(PR_GET_NAME, trace_data.thread_name);
|
||||
|
||||
/* get the thread id */
|
||||
trace_data->tid = syscall(SYS_gettid);
|
||||
trace_data.tid = syscall(SYS_gettid);
|
||||
|
||||
/* Send the output to the main process*/
|
||||
if (write(stacktrace_pipe[1], &trace_data, sizeof(trace_data)) == -1) {/* Avoid warning. */};
|
||||
}
|
||||
|
||||
__attribute__ ((noinline))
|
||||
static void writeStacktraces(int fd, int uplevel) {
|
||||
/* get the list of all the process's threads that don't block or ignore the THREADS_SIGNAL */
|
||||
pid_t pid = getpid();
|
||||
pid_t tids[TIDS_MAX_SIZE];
|
||||
size_t len_tids = get_ready_to_signal_threads_tids(pid, THREADS_SIGNAL, tids);
|
||||
size_t len_tids = get_ready_to_signal_threads_tids(THREADS_SIGNAL, tids);
|
||||
if (!len_tids) {
|
||||
serverLogFromHandler(LL_WARNING, "writeStacktraces(): Failed to get the process's threads.");
|
||||
serverLogRawFromHandler(LL_WARNING, "writeStacktraces(): Failed to get the process's threads.");
|
||||
}
|
||||
|
||||
stacktrace_data stacktraces[len_tids];
|
||||
stacktraces_mempool = stacktraces;
|
||||
memset(stacktraces, 0, sizeof(stacktrace_data) * len_tids);
|
||||
|
||||
/* restart mempool iterator*/
|
||||
atomicSet(g_thread_ids, 0);
|
||||
char buff[PIPE_BUF];
|
||||
/* Clear the stacktraces pipe */
|
||||
while (read(stacktrace_pipe[0], &buff, sizeof(buff)) > 0) {}
|
||||
|
||||
/* ThreadsManager_runOnThreads returns 0 if it is already running */
|
||||
if (!ThreadsManager_runOnThreads(tids, len_tids, collect_stacktrace_data)) return;
|
||||
|
||||
size_t skipped = 0;
|
||||
size_t collected = 0;
|
||||
|
||||
char buff[MAX_BUFF_LENGTH];
|
||||
pid_t calling_tid = syscall(SYS_gettid);
|
||||
/* for backtrace_data in backtraces_data: */
|
||||
for (size_t i = 0; i < len_tids; i++) {
|
||||
stacktrace_data curr_stacktrace_data = stacktraces[i];
|
||||
/*ThreadsManager_runOnThreads might fail to collect the thread's data */
|
||||
if (0 == curr_stacktrace_data.trace_size) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Read the stacktrace_pipe until it's empty */
|
||||
stacktrace_data curr_stacktrace_data = {{0}};
|
||||
while (read(stacktrace_pipe[0], &curr_stacktrace_data, sizeof(curr_stacktrace_data)) > 0) {
|
||||
/* stacktrace header includes the tid and the thread's name */
|
||||
snprintf(buff, MAX_BUFF_LENGTH, "\n%d %s", curr_stacktrace_data.tid, curr_stacktrace_data.thread_name);
|
||||
snprintf_async_signal_safe(buff, sizeof(buff), "\n%d %s", curr_stacktrace_data.tid, curr_stacktrace_data.thread_name);
|
||||
if (write(fd,buff,strlen(buff)) == -1) {/* Avoid warning. */};
|
||||
|
||||
/* skip kernel call to the signal handler, the signal handler and the callback addresses */
|
||||
@ -1882,19 +1878,19 @@ static void writeStacktraces(int fd, int uplevel) {
|
||||
/* skip signal syscall and ThreadsManager_runOnThreads */
|
||||
curr_uplevel += uplevel + 2;
|
||||
/* Add an indication to header of the thread that is handling the log file */
|
||||
snprintf(buff, MAX_BUFF_LENGTH, " *\n");
|
||||
if (write(fd," *\n",strlen(" *\n")) == -1) {/* Avoid warning. */};
|
||||
} else {
|
||||
/* just add a new line */
|
||||
snprintf(buff, MAX_BUFF_LENGTH, "\n");
|
||||
if (write(fd,"\n",strlen("\n")) == -1) {/* Avoid warning. */};
|
||||
}
|
||||
|
||||
if (write(fd,buff,strlen(buff)) == -1) {/* Avoid warning. */};
|
||||
|
||||
/* add the stacktrace */
|
||||
backtrace_symbols_fd(curr_stacktrace_data.trace+curr_uplevel, curr_stacktrace_data.trace_size-curr_uplevel, fd);
|
||||
|
||||
++collected;
|
||||
}
|
||||
|
||||
snprintf(buff, MAX_BUFF_LENGTH, "\n%zu/%zu expected stacktraces.\n", len_tids - skipped, len_tids);
|
||||
snprintf_async_signal_safe(buff, sizeof(buff), "\n%lu/%lu expected stacktraces.\n", (long unsigned)(collected), (long unsigned)len_tids);
|
||||
if (write(fd,buff,strlen(buff)) == -1) {/* Avoid warning. */};
|
||||
|
||||
}
|
||||
@ -1919,7 +1915,7 @@ static void writeStacktraces(int fd, int uplevel) {
|
||||
* Functions that are taken in consideration in "uplevel" should be declared with
|
||||
* __attribute__ ((noinline)) to make sure the compiler won't inline them.
|
||||
*/
|
||||
__attribute__ ((noinline))
|
||||
__attribute__ ((noinline))
|
||||
void logStackTrace(void *eip, int uplevel) {
|
||||
int fd = openDirectLogFiledes();
|
||||
char *msg;
|
||||
@ -1940,6 +1936,9 @@ void logStackTrace(void *eip, int uplevel) {
|
||||
/* Write symbols to log file */
|
||||
++uplevel;
|
||||
writeStacktraces(fd, uplevel);
|
||||
msg = "\n------ STACK TRACE DONE ------\n";
|
||||
if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
|
||||
|
||||
|
||||
/* Cleanup */
|
||||
closeDirectLogFiledes(fd);
|
||||
@ -2215,7 +2214,7 @@ void invalidFunctionWasCalled(void) {}
|
||||
|
||||
typedef void (*invalidFunctionWasCalledType)(void);
|
||||
|
||||
__attribute__ ((noinline))
|
||||
__attribute__ ((noinline))
|
||||
static void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
|
||||
UNUSED(secret);
|
||||
UNUSED(info);
|
||||
@ -2223,7 +2222,7 @@ static void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
|
||||
if(pthread_mutex_lock(&signal_handler_lock) == EDEADLK) {
|
||||
/* If this thread already owns the lock (meaning we crashed during handling a signal)
|
||||
* log that the crash report can't be generated. */
|
||||
serverLog(LL_WARNING,
|
||||
serverLogRawFromHandler(LL_WARNING,
|
||||
"Crashed running signal handler. Can't continue to generate the crash report");
|
||||
/* gracefully exit */
|
||||
bugReportEnd(1, sig);
|
||||
@ -2282,6 +2281,8 @@ static void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
|
||||
}
|
||||
|
||||
void setupDebugSigHandlers(void) {
|
||||
setupStacktracePipe();
|
||||
|
||||
setupSigSegvHandler();
|
||||
|
||||
struct sigaction act;
|
||||
@ -2293,7 +2294,7 @@ void setupDebugSigHandlers(void) {
|
||||
}
|
||||
|
||||
void setupSigSegvHandler(void) {
|
||||
/* Initialize the signal handler lock.
|
||||
/* Initialize the signal handler lock.
|
||||
Attempting to initialize an already initialized mutex or mutexattr results in undefined behavior. */
|
||||
if (!signal_handler_lock_initialized) {
|
||||
/* Set signal handler with error checking attribute. re-lock within the same thread will error. */
|
||||
@ -2355,7 +2356,7 @@ void printCrashReport(void) {
|
||||
void bugReportEnd(int killViaSignal, int sig) {
|
||||
struct sigaction act;
|
||||
|
||||
serverLogRaw(LL_WARNING|LL_RAW,
|
||||
serverLogRawFromHandler(LL_WARNING|LL_RAW,
|
||||
"\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n"
|
||||
" Please report the crash by opening an issue on github:\n\n"
|
||||
" http://github.com/redis/redis/issues\n\n"
|
||||
@ -2424,16 +2425,16 @@ void sigalrmSignalHandler(int sig, siginfo_t *info, void *secret) {
|
||||
/* SIGALRM can be sent explicitly to the process calling kill() to get the stacktraces,
|
||||
or every watchdog_period interval. In the last case, si_pid is not set */
|
||||
if(info->si_pid == 0) {
|
||||
serverLogFromHandler(LL_WARNING,"\n--- WATCHDOG TIMER EXPIRED ---");
|
||||
serverLogRawFromHandler(LL_WARNING,"\n--- WATCHDOG TIMER EXPIRED ---");
|
||||
} else {
|
||||
serverLogFromHandler(LL_WARNING, "\nReceived SIGALRM");
|
||||
serverLogRawFromHandler(LL_WARNING, "\nReceived SIGALRM");
|
||||
}
|
||||
#ifdef HAVE_BACKTRACE
|
||||
logStackTrace(getAndSetMcontextEip(uc, NULL), 1);
|
||||
#else
|
||||
serverLogFromHandler(LL_WARNING,"Sorry: no support for backtrace().");
|
||||
serverLogRawFromHandler(LL_WARNING,"Sorry: no support for backtrace().");
|
||||
#endif
|
||||
serverLogFromHandler(LL_WARNING,"--------\n");
|
||||
serverLogRawFromHandler(LL_WARNING,"--------\n");
|
||||
}
|
||||
|
||||
/* Schedule a SIGALRM delivery after the specified period in milliseconds.
|
||||
@ -2478,30 +2479,41 @@ void debugDelay(int usec) {
|
||||
|
||||
/* =========================== Stacktrace Utils ============================ */
|
||||
|
||||
|
||||
|
||||
/** If it doesn't block and doesn't ignore, return 1 (the thread will handle the signal)
|
||||
* If thread tid blocks or ignores sig_num returns 0 (thread is not ready to catch the signal).
|
||||
* also returns 0 if something is wrong and prints a warning message to the log file **/
|
||||
static int is_thread_ready_to_signal(pid_t pid, pid_t tid, int sig_num) {
|
||||
/* open the threads status file */
|
||||
char buff[MAX_BUFF_LENGTH];
|
||||
snprintf(buff, MAX_BUFF_LENGTH, "/proc/%d/task/%d/status", pid, tid);
|
||||
FILE *thread_status_file = fopen(buff, "r");
|
||||
if (thread_status_file == NULL) {
|
||||
serverLog(LL_WARNING,
|
||||
"tid:%d: failed to open /proc/%d/task/%d/status file", tid, pid, tid);
|
||||
static int is_thread_ready_to_signal(const char *proc_pid_task_path, const char *tid, int sig_num) {
|
||||
/* Open the threads status file path /proc/<pid>>/task/<tid>/status */
|
||||
char path_buff[PATH_MAX];
|
||||
snprintf_async_signal_safe(path_buff, PATH_MAX, "%s/%s/status", proc_pid_task_path, tid);
|
||||
|
||||
int thread_status_file = open(path_buff, O_RDONLY);
|
||||
char buff[PATH_MAX];
|
||||
if (thread_status_file == -1) {
|
||||
serverLogFromHandler(LL_WARNING, "tid:%s: failed to open %s file", tid, path_buff);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ret = 1;
|
||||
size_t field_name_len = strlen("SigBlk:"); /* SigIgn has the same length */
|
||||
size_t field_name_len = strlen("SigBlk:\t"); /* SigIgn has the same length */
|
||||
char *line = NULL;
|
||||
size_t fields_count = 2;
|
||||
while ((line = fgets(buff, MAX_BUFF_LENGTH, thread_status_file)) && fields_count) {
|
||||
while ((line = fgets_async_signal_safe(buff, PATH_MAX, thread_status_file)) && fields_count) {
|
||||
/* iterate the file until we reach SigBlk or SigIgn field line */
|
||||
if (!strncmp(buff, "SigBlk:", field_name_len) || !strncmp(buff, "SigIgn:", field_name_len)) {
|
||||
/* check if the signal exist in the mask */
|
||||
unsigned long sig_mask = strtoul(buff + field_name_len, NULL, 16);
|
||||
if(sig_mask & sig_num) { /* if the signal is blocked/ignored return 0 */
|
||||
if (!strncmp(buff, "SigBlk:\t", field_name_len) || !strncmp(buff, "SigIgn:\t", field_name_len)) {
|
||||
line = buff + field_name_len;
|
||||
unsigned long sig_mask;
|
||||
if (-1 == string2ul_base16_async_signal_safe(line, sizeof(buff), &sig_mask)) {
|
||||
serverLogRawFromHandler(LL_WARNING, "Can't convert signal mask to an unsigned long due to an overflow");
|
||||
ret = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
/* The bit position in a signal mask aligns with the signal number. Since signal numbers start from 1
|
||||
we need to adjust the signal number by subtracting 1 to align it correctly with the zero-based indexing used */
|
||||
if (sig_mask & (1L << (sig_num - 1))) { /* if the signal is blocked/ignored return 0 */
|
||||
ret = 0;
|
||||
break;
|
||||
}
|
||||
@ -2509,58 +2521,84 @@ static int is_thread_ready_to_signal(pid_t pid, pid_t tid, int sig_num) {
|
||||
}
|
||||
}
|
||||
|
||||
fclose(thread_status_file);
|
||||
close(thread_status_file);
|
||||
|
||||
/* if we reached EOF, it means we haven't found SigBlk or/and SigIgn, something is wrong */
|
||||
if (line == NULL) {
|
||||
ret = 0;
|
||||
serverLog(LL_WARNING,
|
||||
"tid:%d: failed to find SigBlk or/and SigIgn field(s) in /proc/%d/task/%d/status file", tid, pid, tid);
|
||||
serverLogFromHandler(LL_WARNING, "tid:%s: failed to find SigBlk or/and SigIgn field(s) in %s/%s/status file", tid, proc_pid_task_path, tid);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** We are using syscall(SYS_getdents64) to read directories, which unlike opendir(), is considered
|
||||
* async-signal-safe. This function wrapper getdents64() in glibc is supported as of glibc 2.30.
|
||||
* To support earlier versions of glibc, we use syscall(SYS_getdents64), which requires defining
|
||||
* linux_dirent64 ourselves. This structure is very old and stable: It will not change unless the kernel
|
||||
* chooses to break compatibility with all existing binaries. Highly Unlikely.
|
||||
*/
|
||||
struct linux_dirent64 {
|
||||
unsigned long long d_ino;
|
||||
long long d_off;
|
||||
unsigned short d_reclen; /* Length of this linux_dirent */
|
||||
unsigned char d_type;
|
||||
char d_name[256]; /* Filename (null-terminated) */
|
||||
};
|
||||
|
||||
/** Returns the number of the process's threads that can receive signal sig_num.
|
||||
* Writes into tids the tids of these threads.
|
||||
* If it fails, returns 0.
|
||||
*/
|
||||
static size_t get_ready_to_signal_threads_tids(pid_t pid, int sig_num, pid_t tids[TIDS_MAX_SIZE]) {
|
||||
/* Initialize the path the process threads' directory. */
|
||||
char path_buff[MAX_BUFF_LENGTH];
|
||||
snprintf(path_buff, MAX_BUFF_LENGTH, "/proc/%d/task", pid);
|
||||
static size_t get_ready_to_signal_threads_tids(int sig_num, pid_t tids[TIDS_MAX_SIZE]) {
|
||||
/* Open /proc/<pid>/task file. */
|
||||
char path_buff[PATH_MAX];
|
||||
snprintf_async_signal_safe(path_buff, PATH_MAX, "/proc/%d/task", getpid());
|
||||
|
||||
/* Get the directory handler. */
|
||||
DIR *dir;
|
||||
if (!(dir = opendir(path_buff))) return 0;
|
||||
int dir;
|
||||
if (-1 == (dir = open(path_buff, O_RDONLY | O_DIRECTORY))) return 0;
|
||||
|
||||
size_t tids_count = 0;
|
||||
struct dirent *entry;
|
||||
pid_t calling_tid = syscall(SYS_gettid);
|
||||
int current_thread_index = -1;
|
||||
long nread;
|
||||
char buff[PATH_MAX];
|
||||
|
||||
/* Each thread is represented by a directory */
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (entry->d_type == DT_DIR) {
|
||||
/* readdir() is not async-signal-safe (AS-safe).
|
||||
Hence, we read the file using SYS_getdents64, which is considered AS-sync*/
|
||||
while ((nread = syscall(SYS_getdents64, dir, buff, PATH_MAX))) {
|
||||
if (nread == -1) {
|
||||
close(dir);
|
||||
serverLogRawFromHandler(LL_WARNING, "get_ready_to_signal_threads_tids(): Failed to read the process's task directory");
|
||||
return 0;
|
||||
}
|
||||
/* Each thread is represented by a directory */
|
||||
for (long pos = 0; pos < nread;) {
|
||||
struct linux_dirent64 *entry = (struct linux_dirent64 *)(buff + pos);
|
||||
pos += entry->d_reclen;
|
||||
/* Skip irrelevant directories. */
|
||||
if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) {
|
||||
/* the thread's directory name is equivalent to its tid. */
|
||||
pid_t tid = atoi(entry->d_name);
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue;
|
||||
|
||||
if(!is_thread_ready_to_signal(pid, tid, sig_num)) continue;
|
||||
/* the thread's directory name is equivalent to its tid. */
|
||||
long tid;
|
||||
string2l(entry->d_name, strlen(entry->d_name), &tid);
|
||||
|
||||
if(tid == calling_tid) {
|
||||
current_thread_index = tids_count;
|
||||
}
|
||||
if(!is_thread_ready_to_signal(path_buff, entry->d_name, sig_num)) continue;
|
||||
|
||||
/* save the thread id */
|
||||
tids[tids_count++] = tid;
|
||||
if(tid == calling_tid) {
|
||||
current_thread_index = tids_count;
|
||||
}
|
||||
|
||||
/* save the thread id */
|
||||
tids[tids_count++] = tid;
|
||||
|
||||
/* Stop if we reached the maximum threads number. */
|
||||
if(tids_count == TIDS_MAX_SIZE) {
|
||||
serverLogRawFromHandler(LL_WARNING, "get_ready_to_signal_threads_tids(): Reached the limit of the tids buffer.");
|
||||
break;
|
||||
}
|
||||
}
|
||||
/* Stop if we reached the maximum threads number. */
|
||||
if(tids_count == TIDS_MAX_SIZE) {
|
||||
serverLogFromHandler(LL_WARNING,"get_ready_to_signal_threads_tids(): Reached the limit of the tids buffer.");
|
||||
break;
|
||||
}
|
||||
|
||||
if(tids_count == TIDS_MAX_SIZE) break;
|
||||
}
|
||||
|
||||
/* Swap the last tid with the the current thread id */
|
||||
@ -2571,7 +2609,7 @@ static size_t get_ready_to_signal_threads_tids(pid_t pid, int sig_num, pid_t tid
|
||||
tids[current_thread_index] = last_tid;
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
close(dir);
|
||||
|
||||
return tids_count;
|
||||
}
|
||||
|
||||
59
src/server.c
59
src/server.c
@ -167,13 +167,9 @@ void _serverLog(int level, const char *fmt, ...) {
|
||||
serverLogRaw(level,msg);
|
||||
}
|
||||
|
||||
/* Log a fixed message without printf-alike capabilities, in a way that is
|
||||
* safe to call from a signal handler.
|
||||
*
|
||||
* We actually use this only for signals that are not fatal from the point
|
||||
* of view of Redis. Signals that are going to kill the server anyway and
|
||||
* where we need printf-alike features are served by serverLog(). */
|
||||
void serverLogFromHandler(int level, const char *msg) {
|
||||
/* Low level logging from signal handler. Should be used with pre-formatted strings.
|
||||
See serverLogFromHandler. */
|
||||
void serverLogRawFromHandler(int level, const char *msg) {
|
||||
int fd;
|
||||
int log_to_stdout = server.logfile[0] == '\0';
|
||||
char buf[64];
|
||||
@ -183,18 +179,41 @@ void serverLogFromHandler(int level, const char *msg) {
|
||||
fd = log_to_stdout ? STDOUT_FILENO :
|
||||
open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644);
|
||||
if (fd == -1) return;
|
||||
ll2string(buf,sizeof(buf),getpid());
|
||||
if (write(fd,buf,strlen(buf)) == -1) goto err;
|
||||
if (write(fd,":signal-handler (",17) == -1) goto err;
|
||||
ll2string(buf,sizeof(buf),time(NULL));
|
||||
if (write(fd,buf,strlen(buf)) == -1) goto err;
|
||||
if (write(fd,") ",2) == -1) goto err;
|
||||
if (write(fd,msg,strlen(msg)) == -1) goto err;
|
||||
if (write(fd,"\n",1) == -1) goto err;
|
||||
if (level & LL_RAW) {
|
||||
if (write(fd,msg,strlen(msg)) == -1) goto err;
|
||||
}
|
||||
else {
|
||||
ll2string(buf,sizeof(buf),getpid());
|
||||
if (write(fd,buf,strlen(buf)) == -1) goto err;
|
||||
if (write(fd,":signal-handler (",17) == -1) goto err;
|
||||
ll2string(buf,sizeof(buf),time(NULL));
|
||||
if (write(fd,buf,strlen(buf)) == -1) goto err;
|
||||
if (write(fd,") ",2) == -1) goto err;
|
||||
if (write(fd,msg,strlen(msg)) == -1) goto err;
|
||||
if (write(fd,"\n",1) == -1) goto err;
|
||||
}
|
||||
err:
|
||||
if (!log_to_stdout) close(fd);
|
||||
}
|
||||
|
||||
/* An async-signal-safe version of serverLog. if LL_RAW is not included in level flags,
|
||||
* The message format is: <pid>:signal-handler (<time>) <msg> \n
|
||||
* with LL_RAW flag only the msg is printed (with no new line at the end)
|
||||
*
|
||||
* We actually use this only for signals that are not fatal from the point
|
||||
* of view of Redis. Signals that are going to kill the server anyway and
|
||||
* where we need printf-alike features are served by serverLog(). */
|
||||
void serverLogFromHandler(int level, const char *fmt, ...) {
|
||||
va_list ap;
|
||||
char msg[LOG_MAX_LEN];
|
||||
|
||||
va_start(ap, fmt);
|
||||
vsnprintf_async_signal_safe(msg, sizeof(msg), fmt, ap);
|
||||
va_end(ap);
|
||||
|
||||
serverLogRawFromHandler(level, msg);
|
||||
}
|
||||
|
||||
/* Return the UNIX time in microseconds */
|
||||
long long ustime(void) {
|
||||
struct timeval tv;
|
||||
@ -774,7 +793,7 @@ int allPersistenceDisabled(void) {
|
||||
/* Add a sample to the instantaneous metric. This function computes the quotient
|
||||
* of the increment of value and base, which is useful to record operation count
|
||||
* per second, or the average time consumption of an operation.
|
||||
*
|
||||
*
|
||||
* current_value - The dividend
|
||||
* current_base - The divisor
|
||||
* */
|
||||
@ -971,7 +990,7 @@ int clientEvictionAllowed(client *c) {
|
||||
|
||||
/* This function is used to cleanup the client's previously tracked memory usage.
|
||||
* This is called during incremental client memory usage tracking as well as
|
||||
* used to reset when client to bucket allocation is not required when
|
||||
* used to reset when client to bucket allocation is not required when
|
||||
* client eviction is disabled. */
|
||||
void removeClientFromMemUsageBucket(client *c, int allow_eviction) {
|
||||
if (c->mem_usage_bucket) {
|
||||
@ -6373,14 +6392,14 @@ static void sigShutdownHandler(int sig) {
|
||||
* the user really wanting to quit ASAP without waiting to persist
|
||||
* on disk and without waiting for lagging replicas. */
|
||||
if (server.shutdown_asap && sig == SIGINT) {
|
||||
serverLogFromHandler(LL_WARNING, "You insist... exiting now.");
|
||||
serverLogRawFromHandler(LL_WARNING, "You insist... exiting now.");
|
||||
rdbRemoveTempFile(getpid(), 1);
|
||||
exit(1); /* Exit with an error since this was not a clean shutdown. */
|
||||
} else if (server.loading) {
|
||||
msg = "Received shutdown signal during loading, scheduling shutdown.";
|
||||
}
|
||||
|
||||
serverLogFromHandler(LL_WARNING, msg);
|
||||
serverLogRawFromHandler(LL_WARNING, msg);
|
||||
server.shutdown_asap = 1;
|
||||
server.last_sig_received = sig;
|
||||
}
|
||||
@ -6404,7 +6423,7 @@ void setupSignalHandlers(void) {
|
||||
static void sigKillChildHandler(int sig) {
|
||||
UNUSED(sig);
|
||||
int level = server.in_fork_child == CHILD_TYPE_MODULE? LL_VERBOSE: LL_WARNING;
|
||||
serverLogFromHandler(level, "Received SIGUSR1 in child, exiting now.");
|
||||
serverLogRawFromHandler(level, "Received SIGUSR1 in child, exiting now.");
|
||||
exitFromChild(SERVER_CHILD_NOERROR_RETVAL);
|
||||
}
|
||||
|
||||
|
||||
@ -3088,11 +3088,14 @@ int mustObeyClient(client *c);
|
||||
#ifdef __GNUC__
|
||||
void _serverLog(int level, const char *fmt, ...)
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
void serverLogFromHandler(int level, const char *fmt, ...)
|
||||
__attribute__((format(printf, 2, 3)));
|
||||
#else
|
||||
void serverLogFromHandler(int level, const char *fmt, ...);
|
||||
void _serverLog(int level, const char *fmt, ...);
|
||||
#endif
|
||||
void serverLogRaw(int level, const char *msg);
|
||||
void serverLogFromHandler(int level, const char *msg);
|
||||
void serverLogRawFromHandler(int level, const char *msg);
|
||||
void usage(void);
|
||||
void updateDictResizePolicy(void);
|
||||
int htNeedsResize(dict *dict);
|
||||
|
||||
@ -32,14 +32,11 @@
|
||||
#define UNUSED(V) ((void) V)
|
||||
|
||||
#ifdef __linux__
|
||||
#include "zmalloc.h"
|
||||
#include "atomicvar.h"
|
||||
#include "server.h"
|
||||
|
||||
#include <signal.h>
|
||||
#include <time.h>
|
||||
#include <errno.h>
|
||||
#include <semaphore.h>
|
||||
#include <sys/syscall.h>
|
||||
|
||||
#define IN_PROGRESS 1
|
||||
@ -51,12 +48,9 @@ static run_on_thread_cb g_callback = NULL;
|
||||
static volatile size_t g_tids_len = 0;
|
||||
static redisAtomic size_t g_num_threads_done = 0;
|
||||
|
||||
static sem_t wait_for_threads_sem;
|
||||
|
||||
/* This flag is set while ThreadsManager_runOnThreads is running */
|
||||
static redisAtomic int g_in_progress = 0;
|
||||
|
||||
static pthread_rwlock_t globals_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
|
||||
/*============================ Internal prototypes ========================== */
|
||||
|
||||
static void invoke_callback(int sig);
|
||||
@ -94,10 +88,10 @@ int ThreadsManager_runOnThreads(pid_t *tids, size_t tids_len, run_on_thread_cb c
|
||||
/* Set g_tids_len */
|
||||
g_tids_len = tids_len;
|
||||
|
||||
/* Initialize a semaphore that we will be waiting on for the threads
|
||||
use pshared = 0 to indicate the semaphore is shared between the process's threads (and not between processes),
|
||||
and value = 0 as the initial semaphore value. */
|
||||
sem_init(&wait_for_threads_sem, 0, 0);
|
||||
/* set g_num_threads_done to 0 To handler the case where in the previous run we reached the timeout
|
||||
and called ThreadsManager_cleanups before one or more threads were done and increased
|
||||
(the already set to 0) g_num_threads_done */
|
||||
g_num_threads_done = 0;
|
||||
|
||||
/* Send signal to all the threads in tids */
|
||||
pid_t pid = getpid();
|
||||
@ -129,55 +123,51 @@ static int test_and_start(void) {
|
||||
__attribute__ ((noinline))
|
||||
static void invoke_callback(int sig) {
|
||||
UNUSED(sig);
|
||||
|
||||
/* If the lock is already locked for write, we are running cleanups, no reason to proceed. */
|
||||
if(0 != pthread_rwlock_tryrdlock(&globals_rw_lock)) {
|
||||
serverLog(LL_WARNING, "threads_mngr: ThreadsManager_cleanups() is in progress, can't invoke signal handler.");
|
||||
return;
|
||||
run_on_thread_cb callback = g_callback;
|
||||
if (callback) {
|
||||
callback();
|
||||
atomicIncr(g_num_threads_done, 1);
|
||||
} else {
|
||||
serverLogFromHandler(LL_WARNING, "tid %ld: ThreadsManager g_callback is NULL", syscall(SYS_gettid));
|
||||
}
|
||||
|
||||
if (g_callback) {
|
||||
g_callback();
|
||||
size_t curr_done_count;
|
||||
atomicIncrGet(g_num_threads_done, curr_done_count, 1);
|
||||
|
||||
/* last thread shuts down the light */
|
||||
if (curr_done_count == g_tids_len) {
|
||||
sem_post(&wait_for_threads_sem);
|
||||
}
|
||||
}
|
||||
|
||||
pthread_rwlock_unlock(&globals_rw_lock);
|
||||
}
|
||||
|
||||
static void wait_threads(void) {
|
||||
struct timespec ts;
|
||||
clock_gettime(CLOCK_REALTIME, &ts);
|
||||
struct timespec timeout_time;
|
||||
clock_gettime(CLOCK_REALTIME, &timeout_time);
|
||||
|
||||
/* calculate relative time until timeout */
|
||||
ts.tv_sec += RUN_ON_THREADS_TIMEOUT;
|
||||
timeout_time.tv_sec += RUN_ON_THREADS_TIMEOUT;
|
||||
|
||||
int status = 0;
|
||||
/* Wait until all threads are done to invoke the callback or until we reached the timeout */
|
||||
size_t curr_done_count;
|
||||
struct timespec curr_time;
|
||||
|
||||
/* lock the semaphore until the semaphore value rises above zero or a signal
|
||||
handler interrupts the call. In the later case continue to wait. */
|
||||
while ((status = sem_timedwait(&wait_for_threads_sem, &ts)) == -1 && errno == EINTR) {
|
||||
serverLog(LL_WARNING, "threads_mngr: waiting for threads' output was interrupted by signal. Continue waiting.");
|
||||
continue;
|
||||
do {
|
||||
struct timeval tv = {
|
||||
.tv_sec = 0,
|
||||
.tv_usec = 10};
|
||||
/* Sleep a bit to yield to other threads. */
|
||||
/* usleep isn't listed as signal safe, so we use select instead */
|
||||
select(0, NULL, NULL, NULL, &tv);
|
||||
atomicGet(g_num_threads_done, curr_done_count);
|
||||
clock_gettime(CLOCK_REALTIME, &curr_time);
|
||||
} while (curr_done_count < g_tids_len &&
|
||||
curr_time.tv_sec <= timeout_time.tv_sec);
|
||||
|
||||
if (curr_time.tv_sec > timeout_time.tv_sec) {
|
||||
serverLogRawFromHandler(LL_WARNING, "wait_threads(): waiting threads timed out");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static void ThreadsManager_cleanups(void) {
|
||||
pthread_rwlock_wrlock(&globals_rw_lock);
|
||||
|
||||
g_callback = NULL;
|
||||
g_tids_len = 0;
|
||||
g_num_threads_done = 0;
|
||||
sem_destroy(&wait_for_threads_sem);
|
||||
|
||||
/* Lastly, turn off g_in_progress */
|
||||
atomicSet(g_in_progress, 0);
|
||||
pthread_rwlock_unlock(&globals_rw_lock);
|
||||
|
||||
}
|
||||
#else
|
||||
|
||||
@ -60,7 +60,7 @@ void ThreadsManager_init(void);
|
||||
* not be signaled until the calling thread returns from the callback invocation.
|
||||
* Hence, it is recommended to place the calling thread last in @param tids.
|
||||
*
|
||||
* The function returns only when @param tids_len threads have returned from @param callback.
|
||||
* The function returns only when @param tids_len threads have returned from @param callback, or when we reached timeout.
|
||||
*
|
||||
* @return 1 if successful, 0 If ThreadsManager_runOnThreads is already in the middle of execution.
|
||||
*
|
||||
|
||||
231
src/util.c
231
src/util.c
@ -1,5 +1,6 @@
|
||||
/*
|
||||
* Copyright (c) 2009-2012, Salvatore Sanfilippo <antirez at gmail dot com>
|
||||
* Copyright (c) 2012, Twitter, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
@ -534,6 +535,43 @@ int string2l(const char *s, size_t slen, long *lval) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* return 1 if c>= start && c <= end, 0 otherwise*/
|
||||
static int safe_is_c_in_range(char c, char start, char end) {
|
||||
if (c >= start && c <= end) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int base_16_char_type(char c) {
|
||||
if (safe_is_c_in_range(c, '0', '9')) return 0;
|
||||
if (safe_is_c_in_range(c, 'a', 'f')) return 1;
|
||||
if (safe_is_c_in_range(c, 'A', 'F')) return 2;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** This is an async-signal safe version of string2l to convert unsigned long to string.
|
||||
* The function translates @param src until it reaches a value that is not 0-9, a-f or A-F, or @param we read slen characters.
|
||||
* On successes writes the result to @param result_output and returns 1.
|
||||
* if the string represents an overflow value, return -1. */
|
||||
int string2ul_base16_async_signal_safe(const char *src, size_t slen, unsigned long *result_output) {
|
||||
static char ascii_to_dec[] = {'0', 'a' - 10, 'A' - 10};
|
||||
|
||||
int char_type = 0;
|
||||
size_t curr_char_idx = 0;
|
||||
unsigned long result = 0;
|
||||
int base = 16;
|
||||
while ((-1 != (char_type = base_16_char_type(src[curr_char_idx]))) &&
|
||||
curr_char_idx < slen) {
|
||||
unsigned long curr_val = src[curr_char_idx] - ascii_to_dec[char_type];
|
||||
if ((result > ULONG_MAX / base) || (result > (ULONG_MAX - curr_val)/base)) /* Overflow. */
|
||||
return -1;
|
||||
result = result * base + curr_val;
|
||||
++curr_char_idx;
|
||||
}
|
||||
|
||||
*result_output = result;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Convert a string into a double. Returns 1 if the string could be parsed
|
||||
* into a (non-overflowing) double, 0 otherwise. The value will be set to
|
||||
* the parsed value when appropriate.
|
||||
@ -1129,7 +1167,7 @@ int fsyncFileDir(const char *filename) {
|
||||
errno = save_errno;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
close(dir_fd);
|
||||
return 0;
|
||||
}
|
||||
@ -1148,6 +1186,195 @@ int reclaimFilePageCache(int fd, size_t offset, size_t length) {
|
||||
#endif
|
||||
}
|
||||
|
||||
/** An async signal safe version of fgets().
|
||||
* Has the same behaviour as standard fgets(): reads a line from fd and stores it into the dest buffer.
|
||||
* It stops when either (buff_size-1) characters are read, the newline character is read, or the end-of-file is reached,
|
||||
* whichever comes first.
|
||||
*
|
||||
* On success, the function returns the same dest parameter. If the End-of-File is encountered and no characters have
|
||||
* been read, the contents of dest remain unchanged and a null pointer is returned.
|
||||
* If an error occurs, a null pointer is returned. */
|
||||
char *fgets_async_signal_safe(char *dest, int buff_size, int fd) {
|
||||
for (int i = 0; i < buff_size; i++) {
|
||||
/* Read one byte */
|
||||
ssize_t bytes_read_count = read(fd, dest + i, 1);
|
||||
/* On EOF or error return NULL */
|
||||
if (bytes_read_count < 1) {
|
||||
return NULL;
|
||||
}
|
||||
/* we found the end of the line. */
|
||||
if (dest[i] == '\n') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
|
||||
static const char HEX[] = "0123456789abcdef";
|
||||
|
||||
static char *u2string_async_signal_safe(int _base, uint64_t val, char *buf) {
|
||||
uint32_t base = (uint32_t) _base;
|
||||
*buf-- = 0;
|
||||
do {
|
||||
*buf-- = HEX[val % base];
|
||||
} while ((val /= base) != 0);
|
||||
return buf + 1;
|
||||
}
|
||||
|
||||
static char *i2string_async_signal_safe(int base, int64_t val, char *buf) {
|
||||
char *orig_buf = buf;
|
||||
const int32_t is_neg = (val < 0);
|
||||
*buf-- = 0;
|
||||
|
||||
if (is_neg) {
|
||||
val = -val;
|
||||
}
|
||||
if (is_neg && base == 16) {
|
||||
int ix;
|
||||
val -= 1;
|
||||
for (ix = 0; ix < 16; ++ix)
|
||||
buf[-ix] = '0';
|
||||
}
|
||||
|
||||
do {
|
||||
*buf-- = HEX[val % base];
|
||||
} while ((val /= base) != 0);
|
||||
|
||||
if (is_neg && base == 10) {
|
||||
*buf-- = '-';
|
||||
}
|
||||
|
||||
if (is_neg && base == 16) {
|
||||
int ix;
|
||||
buf = orig_buf - 1;
|
||||
for (ix = 0; ix < 16; ++ix, --buf) {
|
||||
/* *INDENT-OFF* */
|
||||
switch (*buf) {
|
||||
case '0': *buf = 'f'; break;
|
||||
case '1': *buf = 'e'; break;
|
||||
case '2': *buf = 'd'; break;
|
||||
case '3': *buf = 'c'; break;
|
||||
case '4': *buf = 'b'; break;
|
||||
case '5': *buf = 'a'; break;
|
||||
case '6': *buf = '9'; break;
|
||||
case '7': *buf = '8'; break;
|
||||
case '8': *buf = '7'; break;
|
||||
case '9': *buf = '6'; break;
|
||||
case 'a': *buf = '5'; break;
|
||||
case 'b': *buf = '4'; break;
|
||||
case 'c': *buf = '3'; break;
|
||||
case 'd': *buf = '2'; break;
|
||||
case 'e': *buf = '1'; break;
|
||||
case 'f': *buf = '0'; break;
|
||||
}
|
||||
/* *INDENT-ON* */
|
||||
}
|
||||
}
|
||||
return buf + 1;
|
||||
}
|
||||
|
||||
static const char *check_longlong_async_signal_safe(const char *fmt, int32_t *have_longlong) {
|
||||
*have_longlong = 0;
|
||||
if (*fmt == 'l') {
|
||||
fmt++;
|
||||
if (*fmt != 'l') {
|
||||
*have_longlong = (sizeof(long) == sizeof(int64_t));
|
||||
} else {
|
||||
fmt++;
|
||||
*have_longlong = 1;
|
||||
}
|
||||
}
|
||||
return fmt;
|
||||
}
|
||||
|
||||
int vsnprintf_async_signal_safe(char *to, size_t size, const char *format, va_list ap) {
|
||||
char *start = to;
|
||||
char *end = start + size - 1;
|
||||
for (; *format; ++format) {
|
||||
int32_t have_longlong = 0;
|
||||
if (*format != '%') {
|
||||
if (to == end) { /* end of buffer */
|
||||
break;
|
||||
}
|
||||
*to++ = *format; /* copy ordinary char */
|
||||
continue;
|
||||
}
|
||||
++format; /* skip '%' */
|
||||
|
||||
format = check_longlong_async_signal_safe(format, &have_longlong);
|
||||
|
||||
switch (*format) {
|
||||
case 'd':
|
||||
case 'i':
|
||||
case 'u':
|
||||
case 'x':
|
||||
case 'p':
|
||||
{
|
||||
int64_t ival = 0;
|
||||
uint64_t uval = 0;
|
||||
if (*format == 'p')
|
||||
have_longlong = (sizeof(void *) == sizeof(uint64_t));
|
||||
if (have_longlong) {
|
||||
if (*format == 'u') {
|
||||
uval = va_arg(ap, uint64_t);
|
||||
} else {
|
||||
ival = va_arg(ap, int64_t);
|
||||
}
|
||||
} else {
|
||||
if (*format == 'u') {
|
||||
uval = va_arg(ap, uint32_t);
|
||||
} else {
|
||||
ival = va_arg(ap, int32_t);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
char buff[22];
|
||||
const int base = (*format == 'x' || *format == 'p') ? 16 : 10;
|
||||
|
||||
/* *INDENT-OFF* */
|
||||
char *val_as_str = (*format == 'u') ?
|
||||
u2string_async_signal_safe(base, uval, &buff[sizeof(buff) - 1]) :
|
||||
i2string_async_signal_safe(base, ival, &buff[sizeof(buff) - 1]);
|
||||
/* *INDENT-ON* */
|
||||
|
||||
/* Strip off "ffffffff" if we have 'x' format without 'll' */
|
||||
if (*format == 'x' && !have_longlong && ival < 0) {
|
||||
val_as_str += 8;
|
||||
}
|
||||
|
||||
while (*val_as_str && to < end) {
|
||||
*to++ = *val_as_str++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
case 's':
|
||||
{
|
||||
const char *val = va_arg(ap, char *);
|
||||
if (!val) {
|
||||
val = "(null)";
|
||||
}
|
||||
while (*val && to < end) {
|
||||
*to++ = *val++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
*to = 0;
|
||||
return (int)(to - start);
|
||||
}
|
||||
|
||||
int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...) {
|
||||
int result;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
result = vsnprintf_async_signal_safe(to, n, fmt, args);
|
||||
va_end(args);
|
||||
return result;
|
||||
}
|
||||
|
||||
#ifdef REDIS_TEST
|
||||
#include <assert.h>
|
||||
#include <sys/mman.h>
|
||||
@ -1427,5 +1654,3 @@ int utilTest(int argc, char **argv, int flags) {
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
10
src/util.h
10
src/util.h
@ -70,6 +70,7 @@ int ull2string(char *s, size_t len, unsigned long long value);
|
||||
int string2ll(const char *s, size_t slen, long long *value);
|
||||
int string2ull(const char *s, unsigned long long *value);
|
||||
int string2l(const char *s, size_t slen, long *value);
|
||||
int string2ul_base16_async_signal_safe(const char *src, size_t slen, unsigned long *result_output);
|
||||
int string2ld(const char *s, size_t slen, long double *dp);
|
||||
int string2d(const char *s, size_t slen, double *dp);
|
||||
int trimDoubleString(char *buf, size_t len);
|
||||
@ -88,7 +89,14 @@ int fileExist(char *filename);
|
||||
sds makePath(char *path, char *filename);
|
||||
int fsyncFileDir(const char *filename);
|
||||
int reclaimFilePageCache(int fd, size_t offset, size_t length);
|
||||
|
||||
char *fgets_async_signal_safe(char *dest, int buff_size, int fd);
|
||||
int vsnprintf_async_signal_safe(char *to, size_t size, const char *format, va_list ap);
|
||||
#ifdef __GNUC__
|
||||
int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...)
|
||||
__attribute__((format(printf, 3, 4)));
|
||||
#else
|
||||
int snprintf_async_signal_safe(char *to, size_t n, const char *fmt, ...);
|
||||
#endif
|
||||
size_t redis_strlcpy(char *dst, const char *src, size_t dsize);
|
||||
size_t redis_strlcat(char *dst, const char *src, size_t dsize);
|
||||
|
||||
|
||||
@ -2,7 +2,7 @@ tags {"external:skip"} {
|
||||
|
||||
set system_name [string tolower [exec uname -s]]
|
||||
set backtrace_supported 0
|
||||
set threads_mngr_supported 0
|
||||
set threads_mngr_supported 0 ;# Do we support printing stack trace from all threads, not just the one that got the signal?
|
||||
|
||||
# We only support darwin or Linux with glibc
|
||||
if {$system_name eq {darwin}} {
|
||||
@ -19,12 +19,33 @@ if {$system_name eq {darwin}} {
|
||||
}
|
||||
}
|
||||
|
||||
# look for the DEBUG command in the backtrace, used when we triggered
|
||||
# a stack trace print while we know redis is running that command.
|
||||
proc check_log_backtrace_for_debug {log_pattern} {
|
||||
# search for the final line in the stacktraces generation to make sure it was completed.
|
||||
set pattern "* STACK TRACE DONE *"
|
||||
set res [wait_for_log_messages 0 \"$pattern\" 0 100 100]
|
||||
|
||||
set res [wait_for_log_messages 0 \"$log_pattern\" 0 100 100]
|
||||
if {$::verbose} { puts $res}
|
||||
|
||||
# If the stacktrace is printed more than once, it means redis crashed during crash report generation
|
||||
assert_equal [count_log_message 0 "STACK TRACE"] 1
|
||||
assert_equal [count_log_message 0 "STACK TRACE -"] 1
|
||||
|
||||
upvar threads_mngr_supported threads_mngr_supported
|
||||
|
||||
# the following checks are only done if we support printing stack trace from all threads
|
||||
if {$threads_mngr_supported} {
|
||||
assert_equal [count_log_message 0 "setupStacktracePipe failed"] 0
|
||||
assert_equal [count_log_message 0 "failed to open /proc/"] 0
|
||||
assert_equal [count_log_message 0 "failed to find SigBlk or/and SigIgn"] 0
|
||||
# the following are skipped since valgrind is slow and a timeout can happen
|
||||
if {!$::valgrind} {
|
||||
assert_equal [count_log_message 0 "wait_threads(): waiting threads timed out"] 0
|
||||
# make sure redis prints stack trace for all threads. we know 3 threads are idle in bio.c
|
||||
assert_equal [count_log_message 0 "bioProcessBackgroundJobs"] 3
|
||||
}
|
||||
}
|
||||
|
||||
set pattern "*debugCommand*"
|
||||
set res [wait_for_log_messages 0 \"$pattern\" 0 100 100]
|
||||
@ -44,13 +65,7 @@ if {$backtrace_supported} {
|
||||
test "Server is able to generate a stack trace on selected systems" {
|
||||
r config set watchdog-period 200
|
||||
r debug sleep 1
|
||||
if {$threads_mngr_supported} {
|
||||
assert_equal [count_log_message 0 "failed to open /proc/"] 0
|
||||
assert_equal [count_log_message 0 "failed to find SigBlk or/and SigIgn"] 0
|
||||
assert_equal [count_log_message 0 "threads_mngr: waiting for threads' output was interrupted by signal"] 0
|
||||
assert_equal [count_log_message 0 "threads_mngr: waiting for threads' output timed out"] 0
|
||||
assert_equal [count_log_message 0 "bioProcessBackgroundJobs"] 3
|
||||
}
|
||||
|
||||
check_log_backtrace_for_debug "*WATCHDOG TIMER EXPIRED*"
|
||||
# make sure redis is still alive
|
||||
assert_equal "PONG" [r ping]
|
||||
|
||||
@ -1,13 +1,5 @@
|
||||
set testmodule [file normalize tests/modules/fork.so]
|
||||
|
||||
proc count_log_message {pattern} {
|
||||
set status [catch {exec grep -c $pattern < [srv 0 stdout]} result]
|
||||
if {$status == 1} {
|
||||
set result 0
|
||||
}
|
||||
return $result
|
||||
}
|
||||
|
||||
start_server {tags {"modules"}} {
|
||||
r module load $testmodule
|
||||
|
||||
@ -27,20 +19,20 @@ start_server {tags {"modules"}} {
|
||||
# use a longer time to avoid the child exiting before being killed
|
||||
r fork.create 3 100000000 ;# 100s
|
||||
wait_for_condition 20 100 {
|
||||
[count_log_message "fork child started"] == 2
|
||||
[count_log_message 0 "fork child started"] == 2
|
||||
} else {
|
||||
fail "fork didn't start"
|
||||
}
|
||||
|
||||
# module fork twice
|
||||
assert_error {Fork failed} {r fork.create 0 1}
|
||||
assert {[count_log_message "Can't fork for module: File exists"] eq "1"}
|
||||
assert {[count_log_message 0 "Can't fork for module: File exists"] eq "1"}
|
||||
|
||||
r fork.kill
|
||||
|
||||
assert {[count_log_message "Received SIGUSR1 in child"] eq "1"}
|
||||
assert {[count_log_message 0 "Received SIGUSR1 in child"] eq "1"}
|
||||
# check that it wasn't printed again (the print belong to the previous test)
|
||||
assert {[count_log_message "fork child exiting"] eq "1"}
|
||||
assert {[count_log_message 0 "fork child exiting"] eq "1"}
|
||||
}
|
||||
|
||||
test "Unload the module - fork" {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user