From 3137f26d4c6fe0ca6e9ea4dc9e31eac89ec548e9 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 3 Jun 2018 15:37:48 +0300 Subject: [PATCH 001/117] Add RedisModule_Assert() API call. --- src/module.c | 10 ++++++++++ src/redismodule.h | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/module.c b/src/module.c index 8954fcdf0..c6e6e5989 100644 --- a/src/module.c +++ b/src/module.c @@ -3461,6 +3461,15 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... va_end(ap); } +/* Redis-like assert function. + * + * A failed assertion will shut down the server and produce logging information + * that looks identical to information generated by Redis itself. + */ +void RM__Assert(const char *estr, const char *file, int line) { + _serverAssert(estr, file, line); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -4993,6 +5002,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); + REGISTER_API(_Assert); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); diff --git a/src/redismodule.h b/src/redismodule.h index d18c38881..db32df049 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -271,6 +271,7 @@ void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value) float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str); int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); @@ -433,6 +434,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(EmitAOF); REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); + REDISMODULE_GET_API(_Assert); REDISMODULE_GET_API(StringAppendBuffer); REDISMODULE_GET_API(RetainString); REDISMODULE_GET_API(StringCompare); @@ -499,6 +501,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int return REDISMODULE_OK; } +#define RedisModule_Assert(_e) ((_e)?(void)0 : (RedisModule__Assert(#_e,__FILE__,__LINE__),exit(1))) + #else /* Things only defined for the modules core, not exported to modules From 971d9ee0aa7eb6463f2d95d10c8ecde9a7e3221c Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Tue, 16 Apr 2019 22:16:12 +0300 Subject: [PATCH 002/117] Adds a "Modules" section to `INFO` Fixes #6012. As long as "INFO is broken", this should be adequate IMO. Once we rework `INFO`, perhaps into RESP3, this implementation should be revisited. --- src/module.c | 19 +++++++++++++++++++ src/server.c | 7 +++++++ src/server.h | 1 + 3 files changed, 27 insertions(+) diff --git a/src/module.c b/src/module.c index c29521670..60c9a0464 100644 --- a/src/module.c +++ b/src/module.c @@ -5244,6 +5244,25 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper function for the INFO command: adds loaded modules as to info's + * output. + * + * After the call, the passed sds info string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds genModulesInfoString(sds info) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + sds name = dictGetKey(de); + struct RedisModule *module = dictGetVal(de); + + info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + } + dictReleaseIterator(di); + return info; +} + /* Redis MODULE command. * * MODULE LOAD [args...] */ diff --git a/src/server.c b/src/server.c index fb5d679cd..49a65ef57 100644 --- a/src/server.c +++ b/src/server.c @@ -4291,6 +4291,13 @@ sds genRedisInfoString(char *section) { (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); } + /* Modules */ + if (allsections || defsections || !strcasecmp(section,"modules")) { + if (sections++) info = sdscat(info,"\r\n"); + info = sdscatprintf(info,"# Modules\r\n"); + info = genModulesInfoString(info); + } + /* Command statistics */ if (allsections || !strcasecmp(section,"commandstats")) { if (sections++) info = sdscat(info,"\r\n"); diff --git a/src/server.h b/src/server.h index dfd9f7698..d832c6465 100644 --- a/src/server.h +++ b/src/server.h @@ -2268,6 +2268,7 @@ void bugReportStart(void); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); sds genRedisInfoString(char *section); +sds genModulesInfoString(sds info); void enableWatchdog(int period); void disableWatchdog(void); void watchdogScheduleSignal(int period); From e0387b30f2b3203b64c4b9074d6ae78db4eb165d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 17 Jul 2019 15:08:18 +0300 Subject: [PATCH 003/117] RM_Log - add support for logging without a context or context without module for instance detached thread safe contexts, or various callbacks that don't provide a context. --- src/module.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index f4f753c00..442499b7c 100644 --- a/src/module.c +++ b/src/module.c @@ -3515,7 +3515,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li if (level < server.verbosity) return; - name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); + name_len = snprintf(msg, sizeof(msg),"<%s> ", module? module->name: "module"); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level,msg); } @@ -3533,13 +3533,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li * There is a fixed limit to the length of the log line this function is able * to emit, this limit is not specified but is guaranteed to be more than * a few lines of text. + * + * The ctx argument may be NULL if cannot be provided in the context of the + * caller for instance threads or callbacks, in which case a generic "module" + * will be used instead of the module name. */ void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { - if (!ctx->module) return; /* Can only log if module is initialized */ - va_list ap; va_start(ap, fmt); - RM_LogRaw(ctx->module,levelstr,fmt,ap); + RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap); va_end(ap); } From e70fbad8023e9766dd8580c25aad83a2b0a31be4 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 17 Jul 2019 08:51:02 +0300 Subject: [PATCH 004/117] Module API for Forking * create module API for forking child processes. * refactor duplicate code around creating and tracking forks by AOF and RDB. * child processes listen to SIGUSR1 and dies exitFromChild in order to eliminate a valgrind warning of unhandled signal. * note that BGSAVE error reply has changed. valgrind error is: Process terminating with default action of signal 10 (SIGUSR1) --- runtest-moduleapi | 3 +- src/aof.c | 35 +++--------- src/childinfo.c | 2 + src/db.c | 5 +- src/defrag.c | 2 +- src/module.c | 100 ++++++++++++++++++++++++++++++++++ src/rdb.c | 49 +++-------------- src/redismodule.h | 7 +++ src/replication.c | 6 +- src/scripting.c | 3 +- src/server.c | 90 ++++++++++++++++++++++++++---- src/server.h | 9 +++ tests/modules/Makefile | 6 +- tests/modules/fork.c | 84 ++++++++++++++++++++++++++++ tests/unit/moduleapi/fork.tcl | 32 +++++++++++ 15 files changed, 342 insertions(+), 91 deletions(-) create mode 100644 tests/modules/fork.c create mode 100644 tests/unit/moduleapi/fork.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..da61ab25c 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,5 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork "${@}" + diff --git a/src/aof.c b/src/aof.c index 565ee8073..fc62d86ed 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,9 +264,9 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (server.rdb_child_pid != -1) { + if (hasForkChild() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not @@ -397,7 +397,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + } else if (hasForkChild()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -493,9 +493,8 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && - (server.aof_child_pid != -1 || server.rdb_child_pid != -1)) - return; + if (server.aof_no_fsync_on_rewrite && hasForkChild()) + return; /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { @@ -1562,39 +1561,24 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); serverLog(LL_WARNING, @@ -1608,7 +1592,6 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_rewrite_scheduled = 0; server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; - updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into server.aof_rewrite_buf will start @@ -1623,7 +1606,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (server.rdb_child_pid != -1) { + } else if (hasForkChild()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/childinfo.c b/src/childinfo.c index 719025e8c..fa0600552 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -80,6 +80,8 @@ void receiveChildInfo(void) { server.stat_rdb_cow_bytes = server.child_info_data.cow_size; } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { server.stat_aof_cow_bytes = server.child_info_data.cow_size; + } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) { + server.stat_module_cow_bytes = server.child_info_data.cow_size; } } } diff --git a/src/db.c b/src/db.c index 51f5a12b4..4a489036a 100644 --- a/src/db.c +++ b/src/db.c @@ -60,10 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index ecf0255dc..93c6a4619 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1) + if (hasForkChild()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index f4f753c00..03cc36702 100644 --- a/src/module.c +++ b/src/module.c @@ -30,6 +30,7 @@ #include "server.h" #include "cluster.h" #include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" @@ -291,6 +292,14 @@ typedef struct RedisModuleCommandFilter { /* Registered filters */ static list *moduleCommandFilters; +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); + +static struct RedisModuleForkInfo { + RedisModuleForkDoneHandler done_handler; + void* done_handler_user_data; +} moduleForkInfo = {0}; + + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -5028,6 +5037,94 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Module fork API + * -------------------------------------------------------------------------- */ + +/* Create a background child process with the current frozen snaphost of the + * main process where you can do some processing in the background without + * affecting / freezing the traffic and no need for threads and GIL locking. + * Note that Redis allows for only one concurrent fork. + * When the child wants to exit, it should call RedisModule_ExitFromChild. + * If the parent wants to kill the child it should call RedisModule_KillForkChild + * The done handler callback will be executed on the parent process when the + * child existed (but not when killed) + * Return: -1 on failure, on success the parent process will get a positive PID + * of the child, and the child process will get 0. + */ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) +{ + pid_t childpid; + if (hasForkChild()) { + return -1; + } + + openChildInfoPipe(); + if ((childpid = redisFork()) == 0) { + /* Child */ + redisSetProcTitle("redis-module-fork"); + } else if (childpid == -1) { + closeChildInfoPipe(); + serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno)); + } else { + /* Parent */ + server.module_child_pid = childpid; + moduleForkInfo.done_handler = cb; + moduleForkInfo.done_handler_user_data = user_data; + serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + } + return childpid; +} + +/* Call from the child process when you want to terminate it. + * retcode will be provided to the done handler executed on the parent process. + */ +int RM_ExitFromChild(int retcode) +{ + sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); + exitFromChild(retcode); + return REDISMODULE_OK; +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + int statloc; + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + serverLog(LL_NOTICE,"Killing running module fork child: %ld", + (long) server.module_child_pid); + if (kill(server.module_child_pid,SIGUSR1) != -1) { + while(wait3(&statloc,0,NULL) != server.module_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; + closeChildInfoPipe(); + updateDictResizePolicy(); + return REDISMODULE_OK; +} + +void ModuleForkDoneHandler(int exitcode, int bysignal) +{ + serverLog(LL_NOTICE, + "Module fork exited pid: %d, retcode: %d, bysignal: %d", + server.module_child_pid, exitcode, bysignal); + if (moduleForkInfo.done_handler) { + moduleForkInfo.done_handler(exitcode, bysignal, + moduleForkInfo.done_handler_user_data); + } + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5490,4 +5587,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(Fork); + REGISTER_API(ExitFromChild); + REGISTER_API(KillForkChild); } diff --git a/src/rdb.c b/src/rdb.c index c566378fb..0c3a80d01 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1293,40 +1293,25 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { int retval; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename,rsi); if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); } exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); server.lastbgsave_status = C_ERR; @@ -1338,7 +1323,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_DISK; - updateDictResizePolicy(); return C_OK; } return C_OK; /* unreached */ @@ -2279,10 +2263,9 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { listNode *ln; listIter li; pid_t childpid; - long long start; int pipefds[2]; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2318,8 +2301,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { /* Create the child process. */ openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { /* Child */ int retval; rio slave_sockets; @@ -2327,7 +2309,6 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { rioInitWithFdset(&slave_sockets,fds,numfds); zfree(fds); - closeListeningSockets(0); redisSetProcTitle("redis-rdb-to-slaves"); retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); @@ -2335,16 +2316,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { retval = C_ERR; if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); /* If we are returning OK, at least one slave was served * with the RDB file as expected, so we need to send a report @@ -2413,16 +2385,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { close(pipefds[1]); closeChildInfoPipe(); } else { - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); - serverLog(LL_NOTICE,"Background RDB transfer started by pid %d", childpid); server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; - updateDictResizePolicy(); } zfree(clientids); zfree(fds); @@ -2465,13 +2432,13 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (server.aof_child_pid != -1) { + } else if (hasForkChild()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "An AOF log rewriting in progress: can't BGSAVE right now. " + "Another BG operation is in progress: can't BGSAVE right now. " "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..60681da7c 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -173,6 +173,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value); typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); #define REDISMODULE_TYPE_METHOD_VERSION 1 typedef struct RedisModuleTypeMethods { @@ -357,6 +358,9 @@ const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(R int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos); +int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); +int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); +int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); #endif /* This is included inline inside each Redis module. */ @@ -528,6 +532,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CommandFilterArgInsert); REDISMODULE_GET_API(CommandFilterArgReplace); REDISMODULE_GET_API(CommandFilterArgDelete); + REDISMODULE_GET_API(Fork); + REDISMODULE_GET_API(ExitFromChild); + REDISMODULE_GET_API(KillForkChild); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..7adf5ba38 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,11 +751,11 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (server.aof_child_pid == -1) { + if (!hasForkChild()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, - "No BGSAVE in progress, but an AOF rewrite is active. " + "No BGSAVE in progress, but another BG operation is active. " "BGSAVE for replication delayed"); } } @@ -2899,7 +2899,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/scripting.c b/src/scripting.c index 032bfdf10..deb406457 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1693,7 +1693,7 @@ void ldbSendLogs(void) { int ldbStartSession(client *c) { ldb.forked = (c->flags & CLIENT_LUA_DEBUG_SYNC) == 0; if (ldb.forked) { - pid_t cp = fork(); + pid_t cp = redisFork(); if (cp == -1) { addReplyError(c,"Fork() failed: can't run EVAL in debugging mode."); return 0; @@ -1710,7 +1710,6 @@ int ldbStartSession(client *c) { * socket to make sure if the parent crashes a reset is sent * to the clients. */ serverLog(LL_WARNING,"Redis forked for debugging eval"); - closeListeningSockets(0); } else { /* Parent */ listAddNodeTail(ldb.children,(void*)(unsigned long)cp); diff --git a/src/server.c b/src/server.c index 4337b8f01..675b638d6 100644 --- a/src/server.c +++ b/src/server.c @@ -1447,12 +1447,18 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) + if (!hasForkChild()) dictEnableResize(); else dictDisableResize(); } +int hasForkChild() { + return server.rdb_child_pid != -1 || + server.aof_child_pid != -1 || + server.module_child_pid != -1; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ @@ -1689,7 +1695,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1886,15 +1892,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (server.rdb_child_pid != -1 || server.aof_child_pid != -1 || - ldbPendingChildren()) + if (hasForkChild() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1907,16 +1912,20 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " - "rdb_child_pid = %d, aof_child_pid = %d", + "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", strerror(errno), (int) server.rdb_child_pid, - (int) server.aof_child_pid); + (int) server.aof_child_pid, + (int) server.module_child_pid); } else if (pid == server.rdb_child_pid) { backgroundSaveDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); } else if (pid == server.aof_child_pid) { backgroundRewriteDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); + } else if (pid == server.module_child_pid) { + ModuleForkDoneHandler(exitcode,bysignal); + if (!bysignal && exitcode == 0) receiveChildInfo(); } else { if (!ldbRemoveChild(pid)) { serverLog(LL_WARNING, @@ -1954,8 +1963,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && + !hasForkChild() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2013,7 +2021,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) @@ -2784,6 +2792,7 @@ void initServer(void) { server.cronloops = 0; server.rdb_child_pid = -1; server.aof_child_pid = -1; + server.module_child_pid = -1; server.rdb_child_type = RDB_CHILD_TYPE_NONE; server.rdb_bgsave_scheduled = 0; server.child_info_pipe[0] = -1; @@ -2802,6 +2811,7 @@ void initServer(void) { server.stat_peak_memory = 0; server.stat_rdb_cow_bytes = 0; server.stat_aof_cow_bytes = 0; + server.stat_module_cow_bytes = 0; server.cron_malloc_stats.zmalloc_used = 0; server.cron_malloc_stats.process_rss = 0; server.cron_malloc_stats.allocator_allocated = 0; @@ -4040,7 +4050,9 @@ sds genRedisInfoString(char *section) { "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n" "aof_last_write_status:%s\r\n" - "aof_last_cow_size:%zu\r\n", + "aof_last_cow_size:%zu\r\n" + "module_fork_in_progress:%d\r\n" + "module_fork_last_cow_size:%zu\r\n", server.loading, server.dirty, server.rdb_child_pid != -1, @@ -4058,7 +4070,9 @@ sds genRedisInfoString(char *section) { -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err", (server.aof_last_write_status == C_OK) ? "ok" : "err", - server.stat_aof_cow_bytes); + server.stat_aof_cow_bytes, + server.module_child_pid != -1, + server.stat_module_cow_bytes); if (server.aof_enabled) { info = sdscatprintf(info, @@ -4554,6 +4568,58 @@ void setupSignalHandlers(void) { return; } +static void sigKillChildHandler(int sig) { + UNUSED(sig); + /* this handler is needed to resolve a valgrind warning */ + serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); + exitFromChild(1); +} + +void setupChildSignalHandlers(void) { + struct sigaction act; + + /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction is used. + * Otherwise, sa_handler is used. */ + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = sigKillChildHandler; + sigaction(SIGUSR1, &act, NULL); + return; +} + +int redisFork() { + int childpid; + long long start = ustime(); + if ((childpid = fork()) == 0) { + /* Child */ + closeListeningSockets(0); + setupChildSignalHandlers(); + } else { + /* Parent */ + server.stat_fork_time = ustime()-start; + server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ + latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); + if (childpid == -1) { + return -1; + } + updateDictResizePolicy(); + } + return childpid; +} + +void sendChildCOWInfo(int ptype, char *pname) { + size_t private_dirty = zmalloc_get_private_dirty(-1); + + if (private_dirty) { + serverLog(LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, private_dirty/(1024*1024)); + } + + server.child_info_data.cow_size = private_dirty; + sendChildInfo(ptype); +} + void memtest(size_t megabytes, int passes); /* Returns 1 if there is --sentinel among the arguments or if diff --git a/src/server.h b/src/server.h index f81b1010e..deaaf2636 100644 --- a/src/server.h +++ b/src/server.h @@ -1030,6 +1030,7 @@ struct clusterState; #define CHILD_INFO_MAGIC 0xC17DDA7A12345678LL #define CHILD_INFO_TYPE_RDB 0 #define CHILD_INFO_TYPE_AOF 1 +#define CHILD_INFO_TYPE_MODULE 3 struct redisServer { /* General */ @@ -1065,6 +1066,7 @@ struct redisServer { int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs to be processed. */ + pid_t module_child_pid; /* PID of module child */ /* Networking */ int port; /* TCP listening port */ int tcp_backlog; /* TCP listen() backlog */ @@ -1138,6 +1140,7 @@ struct redisServer { _Atomic long long stat_net_output_bytes; /* Bytes written to network. */ size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ + size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ struct { @@ -1528,6 +1531,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +void ModuleForkDoneHandler(int exitcode, int bysignal); /* Utils */ long long ustime(void); @@ -1786,6 +1790,11 @@ void closeChildInfoPipe(void); void sendChildInfo(int process_type); void receiveChildInfo(void); +/* Fork helpers */ +int redisFork(); +int hasForkChild(); +void sendChildCOWInfo(int ptype, char *pname); + /* acl.c -- Authentication related prototypes. */ extern rax *Users; extern user *DefaultUser; diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20afa..846d4c87d 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,16 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so fork.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +fork.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +fork.so: fork.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/fork.c b/tests/modules/fork.c new file mode 100644 index 000000000..0804e4355 --- /dev/null +++ b/tests/modules/fork.c @@ -0,0 +1,84 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include + +#define UNUSED(V) ((void) V) + +int child_pid = -1; +int exitted_with_code = -1; + +void done_handler(int exitcode, int bysignal, void *user_data) { + child_pid = -1; + exitted_with_code = exitcode; + assert(user_data==(void*)0xdeadbeef); + UNUSED(bysignal); +} + +int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + long long code_to_exit_with; + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModule_StringToLongLong(argv[1], &code_to_exit_with); + exitted_with_code = -1; + child_pid = RedisModule_Fork(done_handler, (void*)0xdeadbeef); + if (child_pid < 0) { + RedisModule_ReplyWithError(ctx, "Fork failed"); + return REDISMODULE_OK; + } else if (child_pid > 0) { + /* parent */ + RedisModule_ReplyWithLongLong(ctx, child_pid); + return REDISMODULE_OK; + } + + /* child */ + RedisModule_Log(ctx, "notice", "fork child started"); + usleep(200000); + RedisModule_Log(ctx, "notice", "fork child exiting"); + RedisModule_ExitFromChild(code_to_exit_with); + /* unreachable */ + return 0; +} + +int fork_exitcode(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + RedisModule_ReplyWithLongLong(ctx, exitted_with_code); + return REDISMODULE_OK; +} + +int fork_kill(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + if (RedisModule_KillForkChild(child_pid) != REDISMODULE_OK) + RedisModule_ReplyWithError(ctx, "KillForkChild failed"); + else + RedisModule_ReplyWithLongLong(ctx, 1); + child_pid = -1; + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + if (RedisModule_Init(ctx,"fork",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.create", fork_create,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.exitcode", fork_exitcode,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.kill", fork_kill,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl new file mode 100644 index 000000000..f7d7e47d5 --- /dev/null +++ b/tests/unit/moduleapi/fork.tcl @@ -0,0 +1,32 @@ +set testmodule [file normalize tests/modules/fork.so] + +proc count_log_message {pattern} { + set result [exec grep -c $pattern < [srv 0 stdout]] +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module fork} { + # the argument to fork.create is the exitcode on termination + r fork.create 3 + wait_for_condition 20 100 { + [r fork.exitcode] != -1 + } else { + fail "fork didn't terminate" + } + r fork.exitcode + } {3} + + test {Module fork kill} { + r fork.create 3 + after 20 + r fork.kill + after 100 + + assert {[count_log_message "fork child started"] eq "2"} + assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} + assert {[count_log_message "fork child exiting"] eq "1"} + } + +} From c80ad2f4b53c3345f565e86fbdbcac9403ad094c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 14 Jul 2019 17:34:13 +0300 Subject: [PATCH 005/117] Allow modules to handle RDB loading errors. This is especially needed in diskless loading, were a short read could have caused redis to exit. now the module can handle the error and return to the caller gracefully. this fixes #5326 --- src/module.c | 41 +++++++++++++++++++++++++++++++++++------ src/redismodule.h | 9 +++++++++ src/server.h | 2 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index f4f753c00..68952b1e0 100644 --- a/src/module.c +++ b/src/module.c @@ -3139,9 +3139,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * RDB loading and saving functions * -------------------------------------------------------------------------- */ -/* Called when there is a load error in the context of a module. This cannot - * be recovered like for the built-in types. */ +/* Called when there is a load error in the context of a module. On some + * modules this cannot be recovered, but if the module declared capability + * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { + if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + io->error = 1; + return; + } serverLog(LL_WARNING, "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " @@ -3152,6 +3157,23 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } +/* Set flags defining capabilities or behavior */ +void RM_SetIOFlags(RedisModuleIO *io, int flags) { + io->flags = flags; +} + +/* Get flags which were set by RedisModule_SetIOFlags */ +int RM_GetIOFlags(RedisModuleIO *io) { + return io->flags; +} + +/* Returns true if any previous IO API failed. + * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetIOFlags first. */ +int RM_IsIOError(RedisModuleIO *io) { + return io->error; +} + /* Save an unsigned 64 bit value into the RDB file. This function should only * be called in the context of the rdb_save method of modules implementing new * data types. */ @@ -3175,6 +3197,7 @@ saveerr: * be called in the context of the rdb_load method of modules implementing * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; @@ -3186,7 +3209,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ @@ -3245,6 +3268,7 @@ saveerr: /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { + if (io->error) return NULL; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; @@ -3256,7 +3280,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { loaderr: moduleRDBLoadError(io); - return NULL; /* Never reached. */ + return NULL; } /* In the context of the rdb_load method of a module data type, loads a string @@ -3305,6 +3329,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; @@ -3316,7 +3341,7 @@ double RM_LoadDouble(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* In the context of the rdb_save method of a module data type, saves a float @@ -3341,6 +3366,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; @@ -3352,7 +3378,7 @@ float RM_LoadFloat(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* -------------------------------------------------------------------------- @@ -5408,6 +5434,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); + REGISTER_API(GetIOFlags); + REGISTER_API(SetIOFlags); + REGISTER_API(IsIOError); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..154b5b405 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,6 +140,9 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) +/* Declare that the module can handle errors with RedisModule_SetIOFlags */ +#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE @@ -271,6 +274,9 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); +void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); +int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); +int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -444,6 +450,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); + REDISMODULE_GET_API(SetIOFlags); + REDISMODULE_GET_API(GetIOFlags); + REDISMODULE_GET_API(IsIOError); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/server.h b/src/server.h index f81b1010e..9957c3b5c 100644 --- a/src/server.h +++ b/src/server.h @@ -599,6 +599,7 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ + int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -611,6 +612,7 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ + iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis From 4b89b52355947e2b7aba444139575a575f994fa2 Mon Sep 17 00:00:00 2001 From: chendianqiang Date: Mon, 22 Jul 2019 17:45:30 +0800 Subject: [PATCH 006/117] make memory usage consistent of robj with OBJ_ENCODING_INT --- src/object.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index ec0bd02ee..43f46b020 100644 --- a/src/object.c +++ b/src/object.c @@ -455,10 +455,15 @@ robj *tryObjectEncoding(robj *o) { incrRefCount(shared.integers[value]); return shared.integers[value]; } else { - if (o->encoding == OBJ_ENCODING_RAW) sdsfree(o->ptr); - o->encoding = OBJ_ENCODING_INT; - o->ptr = (void*) value; - return o; + if (o->encoding == OBJ_ENCODING_RAW) { + sdsfree(o->ptr); + o->encoding = OBJ_ENCODING_INT; + o->ptr = (void*) value; + return o; + } else { + decrRefCount(o); + return createStringObjectFromLongLongForValue(value); + } } } From bcf3cf8832940517fa73f30f7a35dcb5d4614943 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 11:47:44 +0200 Subject: [PATCH 007/117] Clinet side caching: take count of used caching slots. --- src/tracking.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 84b29d28a..5938efade 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -60,6 +60,7 @@ * use the most significant bits instead of the full 24 bits. */ #define TRACKING_TABLE_SIZE (1<<24) rax **TrackingTable = NULL; +unsigned long TrackingTableUsedSlots = 0; robj *TrackingChannelName; /* Remove the tracking state from the client 'c'. Note that there is not much @@ -109,8 +110,10 @@ void trackingRememberKeys(client *c) { sds sdskey = c->argv[idx]->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) + if (TrackingTable[hash] == NULL) { TrackingTable[hash] = raxNew(); + TrackingTableUsedSlots++; + } raxTryInsert(TrackingTable[hash], (unsigned char*)&c->id,sizeof(c->id),NULL,NULL); } @@ -176,6 +179,7 @@ void trackingInvalidateKey(robj *keyobj) { * again if more keys will be modified in this hash slot. */ raxFree(TrackingTable[hash]); TrackingTable[hash] = NULL; + TrackingTableUsedSlots--; } void trackingInvalidateKeysOnFlush(int dbid) { From ab490723629cd50ab49cc699023c5af31ab427e7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:10:51 +0200 Subject: [PATCH 008/117] Client side caching: don't hash the key if not needed. --- src/tracking.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 5938efade..7ee4d0b2c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -156,12 +156,15 @@ void sendTrackingMessage(client *c, long long hash) { /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is - * to send a notification to every client that may have keys about such . */ + * to send a notification to every client that may have keys about such caching + * slot. */ void trackingInvalidateKey(robj *keyobj) { + if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return; + sds sdskey = keyobj->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable == NULL || TrackingTable[hash] == NULL) return; + if (TrackingTable[hash] == NULL) return; raxIterator ri; raxStart(&ri,TrackingTable[hash]); From df2e7cf4fc93055542175207055f4c43aaa57880 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:29:54 +0200 Subject: [PATCH 009/117] Client side caching: call the invalidation functions always. Otherwise what happens is that the tracking table will never get garbage collected if there are no longer clients with tracking enabled. Now the invalidation function immediately checks if there is any table allocated, otherwise it returns ASAP, so the overhead when the feature is not used should be near zero. --- src/db.c | 4 ++-- src/expire.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 568e0b8de..cc31a117b 100644 --- a/src/db.c +++ b/src/db.c @@ -412,12 +412,12 @@ long long dbTotalServerKeyCount() { void signalModifiedKey(redisDb *db, robj *key) { touchWatchedKey(db,key); - if (server.tracking_clients) trackingInvalidateKey(key); + trackingInvalidateKey(key); } void signalFlushedDb(int dbid) { touchWatchedKeysOnFlush(dbid); - if (server.tracking_clients) trackingInvalidateKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(dbid); } /*----------------------------------------------------------------------------- diff --git a/src/expire.c b/src/expire.c index b23117a3c..598b27f96 100644 --- a/src/expire.c +++ b/src/expire.c @@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { dbSyncDelete(db,keyobj); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",keyobj,db->id); - if (server.tracking_clients) trackingInvalidateKey(keyobj); + trackingInvalidateKey(keyobj); decrRefCount(keyobj); server.stat_expiredkeys++; return 1; From 06de8790dcf95d0e724e06c0f6f164c1fb2c08e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:31:46 +0200 Subject: [PATCH 010/117] Client side caching: reclaim the tracking table on FLUSHALL. --- src/tracking.c | 52 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 7ee4d0b2c..dc2934831 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -185,17 +185,53 @@ void trackingInvalidateKey(robj *keyobj) { TrackingTableUsedSlots--; } +/* This function is called when one or all the Redis databases are flushed + * (dbid == -1 in case of FLUSHALL). Caching slots are not specific for + * each DB but are global: currently what we do is sending a special + * notification to clients with tracking enabled, invalidating the caching + * slot "-1", which means, "all the keys", in order to avoid flooding clients + * with many invalidation messages for all the keys they may hold. + * + * However trying to flush the tracking table here is very costly: + * we need scanning 16 million caching slots in the table to check + * if they are used, this introduces a big delay. So what we do is to really + * flush the table in the case of FLUSHALL. When a FLUSHDB is called instead + * we just send the invalidation message to all the clients, but don't + * flush the table: it will slowly get garbage collected as more keys + * are modified in the used caching slots. */ void trackingInvalidateKeysOnFlush(int dbid) { UNUSED(dbid); - if (server.tracking_clients == 0) return; - listNode *ln; - listIter li; - listRewind(server.clients,&li); - while ((ln = listNext(&li)) != NULL) { - client *c = listNodeValue(ln); - if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,-1); + if (server.tracking_clients) { + listNode *ln; + listIter li; + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (c->flags & CLIENT_TRACKING) { + sendTrackingMessage(c,-1); + } + } + } + + /* In case of FLUSHALL, reclaim all the memory used by tracking. */ + if (dbid == -1 && TrackingTable) { + for (int j = 0; j < TRACKING_TABLE_SIZE; j++) { + if (TrackingTable[j] != NULL) { + raxFree(TrackingTable[j]); + TrackingTable[j] = NULL; + TrackingTableUsedSlots--; + if (TrackingTableUsedSlots == 0) break; + } + } + + /* If there are no clients with tracking enabled, we can even + * reclaim the memory used by the table itself. The code assumes + * the table is allocated only if there is at least one client alive + * with tracking enabled. */ + if (server.tracking_clients == 0) { + zfree(TrackingTable); + TrackingTable = NULL; } } } From ac05029790f88e5897275773e26cb9acb7dc2786 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:37:43 +0200 Subject: [PATCH 011/117] Move signalFlushedDb() into a better place. Now that the call also invalidates client side caching slots, it is important that after an internal flush operation we both send the notifications to the clients and, at the same time, are able to reclaim the memory of the tracking table. This may even fix a few edge cases related to MULTI/EXEC + WATCH during resync, not sure, but in general looks more correct. --- src/db.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index cc31a117b..3d8b844e8 100644 --- a/src/db.c +++ b/src/db.c @@ -378,6 +378,7 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( } } if (dbnum == -1) flushSlaveKeysWithExpireList(); + signalFlushedDb(dbnum); return removed; } @@ -453,7 +454,6 @@ void flushdbCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); } @@ -465,7 +465,6 @@ void flushallCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(-1); server.dirty += emptyDb(-1,flags,NULL); addReply(c,shared.ok); if (server.rdb_child_pid != -1) killRDBChild(); From 023b67e80f23921da305c0c6a4615f7bb95d9049 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 18:45:47 +0200 Subject: [PATCH 012/117] Client side caching: redis-cli ability to enable tracking. This is extremely useful in order to simulate an high load of requests about different keys, and force Redis to track a lot of informations about several clients, to simulate real world workloads. --- src/redis-benchmark.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 1d16fa4ee..2df41580b 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -104,6 +104,7 @@ static struct config { int is_fetching_slots; int is_updating_slots; int slots_last_update; + int enable_tracking; /* Thread mutexes to be used as fallbacks by atomicvar.h */ pthread_mutex_t requests_issued_mutex; pthread_mutex_t requests_finished_mutex; @@ -255,7 +256,7 @@ static redisConfig *getRedisConfig(const char *ip, int port, goto fail; } - if(config.auth){ + if(config.auth) { void *authReply = NULL; redisAppendCommand(c, "AUTH %s", config.auth); if (REDIS_OK != redisGetReply(c, &authReply)) goto fail; @@ -633,6 +634,14 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) { c->prefix_pending++; } + if (config.enable_tracking) { + char *buf = NULL; + int len = redisFormatCommand(&buf, "CLIENT TRACKING on"); + c->obuf = sdscatlen(c->obuf, buf, len); + free(buf); + c->prefix_pending++; + } + /* If a DB number different than zero is selected, prefix our request * buffer with the SELECT command, that will be discarded the first * time the replies are received, so if the client is reused the @@ -1350,6 +1359,8 @@ int parseOptions(int argc, const char **argv) { } else if (config.num_threads < 0) config.num_threads = 0; } else if (!strcmp(argv[i],"--cluster")) { config.cluster_mode = 1; + } else if (!strcmp(argv[i],"--enable-tracking")) { + config.enable_tracking = 1; } else if (!strcmp(argv[i],"--help")) { exit_status = 0; goto usage; @@ -1380,6 +1391,7 @@ usage: " --dbnum SELECT the specified db number (default 0)\n" " --threads Enable multi-thread mode.\n" " --cluster Enable cluster mode.\n" +" --enable-tracking Send CLIENT TRACKING on before starting benchmark.\n" " -k 1=keep alive 0=reconnect (default 1)\n" " -r Use random keys for SET/GET/INCR, random values for SADD\n" " Using this option the benchmark will expand the string __rand_int__\n" @@ -1504,6 +1516,7 @@ int main(int argc, const char **argv) { config.is_fetching_slots = 0; config.is_updating_slots = 0; config.slots_last_update = 0; + config.enable_tracking = 0; i = parseOptions(argc,argv); argc -= i; From f2f3d91e781af95f3163f2322f6b5ef364afdfd7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 18:59:53 +0200 Subject: [PATCH 013/117] Client side caching: split invalidation into key / slot. --- src/server.c | 4 ++++ src/server.h | 1 + src/tracking.c | 58 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/server.c b/src/server.c index 4337b8f01..5e18b5465 100644 --- a/src/server.c +++ b/src/server.c @@ -3401,6 +3401,10 @@ int processCommand(client *c) { } } + /* Make sure to use a reasonable amount of memory for client side + * caching metadata. */ + if (server.tracking_clients) trackingLimitUsedSlots(); + /* Don't accept write commands if there are problems persisting on disk * and if this is a master instance. */ int deny_write_type = writeCommandsDeniedByDiskError(); diff --git a/src/server.h b/src/server.h index b200a6696..fdab53830 100644 --- a/src/server.h +++ b/src/server.h @@ -1639,6 +1639,7 @@ void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); +void trackingLimitUsedSlots(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index dc2934831..1189e82d5 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -154,6 +154,30 @@ void sendTrackingMessage(client *c, long long hash) { } } +/* Invalidates a caching slot: this is actually the low level implementation + * of the API that Redis calls externally, that is trackingInvalidateKey(). */ +void trackingInvalidateSlot(uint64_t slot) { + if (TrackingTable == NULL || TrackingTable[slot] == NULL) return; + + raxIterator ri; + raxStart(&ri,TrackingTable[slot]); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + uint64_t id; + memcpy(&id,ri.key,ri.key_len); + client *c = lookupClientByID(id); + if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + sendTrackingMessage(c,slot); + } + raxStop(&ri); + + /* Free the tracking table: we'll create the radix tree and populate it + * again if more keys will be modified in this caching slot. */ + raxFree(TrackingTable[slot]); + TrackingTable[slot] = NULL; + TrackingTableUsedSlots--; +} + /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is * to send a notification to every client that may have keys about such caching @@ -164,25 +188,7 @@ void trackingInvalidateKey(robj *keyobj) { sds sdskey = keyobj->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) return; - - raxIterator ri; - raxStart(&ri,TrackingTable[hash]); - raxSeek(&ri,"^",NULL,0); - while(raxNext(&ri)) { - uint64_t id; - memcpy(&id,ri.key,ri.key_len); - client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,hash); - } - raxStop(&ri); - - /* Free the tracking table: we'll create the radix tree and populate it - * again if more keys will be modified in this hash slot. */ - raxFree(TrackingTable[hash]); - TrackingTable[hash] = NULL; - TrackingTableUsedSlots--; + trackingInvalidateSlot(hash); } /* This function is called when one or all the Redis databases are flushed @@ -235,3 +241,17 @@ void trackingInvalidateKeysOnFlush(int dbid) { } } } + +/* Tracking forces Redis to remember information about which client may have + * keys about certian caching slots. In workloads where there are a lot of + * reads, but keys are hardly modified, the amount of information we have + * to remember server side could be a lot: for each 16 millions of caching + * slots we may end with a radix tree containing many entries. + * + * So Redis allows the user to configure a maximum fill rate for the + * invalidation table. This function makes sure that we don't go over the + * specified fill rate: if we are over, we can just evict informations about + * random caching slots, and send invalidation messages to clients like if + * the key was modified. */ +void trackingLimitUsedSlots(void) { +} From ff0780e8e6a3101666a70e28952163eff648fa5c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 21 Jul 2019 17:41:03 +0300 Subject: [PATCH 014/117] Implement module api for aux data in rdb Other changes: * fix memory leak in error handling of rdb loading of type OBJ_MODULE --- runtest-moduleapi | 2 +- src/module.c | 41 ++++++ src/rdb.c | 87 ++++++++++-- src/rdb.h | 1 + src/redismodule.h | 11 +- src/server.h | 10 ++ tests/modules/Makefile | 6 +- tests/modules/testrdb.c | 229 +++++++++++++++++++++++++++++++ tests/unit/moduleapi/testrdb.tcl | 62 +++++++++ 9 files changed, 431 insertions(+), 18 deletions(-) create mode 100644 tests/modules/testrdb.c create mode 100644 tests/unit/moduleapi/testrdb.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..8e1c0cb23 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}" diff --git a/src/module.c b/src/module.c index f4f753c00..812a54e04 100644 --- a/src/module.c +++ b/src/module.c @@ -29,6 +29,7 @@ #include "server.h" #include "cluster.h" +#include "rdb.h" #include #define REDISMODULE_CORE 1 @@ -3078,6 +3079,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; + struct { + moduleTypeAuxLoadFunc aux_load; + moduleTypeAuxSaveFunc aux_save; + int aux_save_triggers; + } v2; } *tms = (struct typemethods*) typemethods_ptr; moduleType *mt = zcalloc(sizeof(*mt)); @@ -3089,6 +3095,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, mt->mem_usage = tms->mem_usage; mt->digest = tms->digest; mt->free = tms->free; + if (tms->version >= 2) { + mt->aux_load = tms->v2.aux_load; + mt->aux_save = tms->v2.aux_save; + mt->aux_save_triggers = tms->v2.aux_save_triggers; + } memcpy(mt->name,name,sizeof(mt->name)); listAddNodeTail(ctx->module->types,mt); return mt; @@ -3355,6 +3366,36 @@ loaderr: return 0; /* Never reached. */ } +/* Iterate over modules, and trigger rdb aux saving for the ones modules types + * who asked for it. */ +ssize_t rdbSaveModulesAux(rio *rdb, int when) { + size_t total_written = 0; + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + listIter li; + listNode *ln; + + listRewind(module->types,&li); + while((ln = listNext(&li))) { + moduleType *mt = ln->value; + if (!mt->aux_save || !(mt->aux_save_triggers & when)) + continue; + ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt); + if (ret==-1) { + dictReleaseIterator(di); + return -1; + } + total_written += ret; + } + } + + dictReleaseIterator(di); + return total_written; +} + /* -------------------------------------------------------------------------- * Key digest API (DEBUG DIGEST interface for modules types) * -------------------------------------------------------------------------- */ diff --git a/src/rdb.c b/src/rdb.c index 2118a00f4..4e00fad67 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -971,7 +971,6 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { RedisModuleIO io; moduleValue *mv = o->ptr; moduleType *mt = mv->type; - moduleInitIOContext(io,mt,rdb,key); /* Write the "module" identifier as prefix, so that we'll be able * to call the right module during loading. */ @@ -980,10 +979,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { io.bytes += retval; /* Then write the module-specific representation + EOF marker. */ + moduleInitIOContext(io,mt,rdb,key); mt->rdb_save(&io,mv->value); retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF); - if (retval == -1) return -1; - io.bytes += retval; + if (retval == -1) + io.error = 1; + else + io.bytes += retval; if (io.ctx) { moduleFreeContext(io.ctx); @@ -1101,6 +1103,40 @@ int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) { return 1; } +ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) { + /* Save a module-specific aux value. */ + RedisModuleIO io; + int retval = rdbSaveType(rdb, RDB_OPCODE_MODULE_AUX); + + /* Write the "module" identifier as prefix, so that we'll be able + * to call the right module during loading. */ + retval = rdbSaveLen(rdb,mt->id); + if (retval == -1) return -1; + io.bytes += retval; + + /* write the 'when' so that we can provide it on loading */ + retval = rdbSaveLen(rdb,when); + if (retval == -1) return -1; + io.bytes += retval; + + /* Then write the module-specific representation + EOF marker. */ + moduleInitIOContext(io,mt,rdb,NULL); + mt->aux_save(&io,when); + retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF); + if (retval == -1) + io.error = 1; + else + io.bytes += retval; + + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + if (io.error) + return -1; + return io.bytes; +} + /* Produces a dump of the database in RDB format sending it to the specified * Redis I/O channel. On success C_OK is returned, otherwise C_ERR * is returned and part of the output, or all the output, can be @@ -1122,6 +1158,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION); if (rdbWriteRaw(rdb,magic,9) == -1) goto werr; if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr; + if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; @@ -1183,6 +1220,8 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { di = NULL; /* So that we don't release it again on error. */ } + if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_AFTER_RDB) == -1) goto werr; + /* EOF opcode */ if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr; @@ -2089,15 +2128,11 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { decrRefCount(auxval); continue; /* Read type again. */ } else if (type == RDB_OPCODE_MODULE_AUX) { - /* This is just for compatibility with the future: we have plans - * to add the ability for modules to store anything in the RDB - * file, like data that is not related to the Redis key space. - * Such data will potentially be stored both before and after the - * RDB keys-values section. For this reason since RDB version 9, - * we have the ability to read a MODULE_AUX opcode followed by an - * identifier of the module, and a serialized value in "MODULE V2" - * format. */ + /* Load module data that is not related to the Redis key space. + * Such data can be potentially be stored both before and after the + * RDB keys-values section. */ uint64_t moduleid = rdbLoadLen(rdb,NULL); + int when = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) goto eoferr; moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; @@ -2108,10 +2143,32 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load: no matching module '%s'", name); exit(1); } else if (!rdbCheckMode && mt != NULL) { - /* This version of Redis actually does not know what to do - * with modules AUX data... */ - serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load for the module '%s'. Probably you want to use a newer version of Redis which implements aux data callbacks", name); - exit(1); + if (!mt->aux_load) { + /* Module doesn't support AUX. */ + serverLog(LL_WARNING,"The RDB file contains module AUX data, but the module '%s' doesn't seem to support it.", name); + exit(1); + } + + RedisModuleIO io; + moduleInitIOContext(io,mt,rdb,NULL); + io.ver = 2; + /* Call the rdb_load method of the module providing the 10 bit + * encoding version in the lower 10 bits of the module ID. */ + if (mt->aux_load(&io,moduleid&1023, when) || io.error) { + moduleTypeNameByID(name,moduleid); + serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); + exit(1); + } + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + uint64_t eof = rdbLoadLen(rdb,NULL); + if (eof != RDB_MODULE_OPCODE_EOF) { + serverLog(LL_WARNING,"The RDB file contains module AUX data for the module '%s' that is not terminated by the proper module value EOF marker", name); + exit(1); + } + continue; } else { /* RDB check mode. */ robj *aux = rdbLoadCheckModuleValue(rdb,name); diff --git a/src/rdb.h b/src/rdb.h index 0acddf9ab..40a50f7ba 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -145,6 +145,7 @@ size_t rdbSavedObjectLen(robj *o); robj *rdbLoadObject(int type, rio *rdb, robj *key); void backgroundSaveDoneHandler(int exitcode, int bysignal); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); +ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt); robj *rdbLoadStringObject(rio *rdb); ssize_t rdbSaveStringObject(rio *rdb, robj *obj); ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len); diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..21227fe3a 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -129,6 +129,10 @@ #define REDISMODULE_NOT_USED(V) ((void) V) +/* Bit flags for aux_save_triggers and the aux_load and aux_save callbacks */ +#define REDISMODULE_AUX_BEFORE_RDB (1<<0) +#define REDISMODULE_AUX_AFTER_RDB (1<<1) + /* This type represents a timer handle, and is returned when a timer is * registered and used in order to invalidate a timer. It's just a 64 bit * number, because this is how each timer is represented inside the radix tree @@ -166,6 +170,8 @@ typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlocke typedef int (*RedisModuleNotificationFunc)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver); typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value); +typedef int (*RedisModuleTypeAuxLoadFunc)(RedisModuleIO *rdb, int encver, int when); +typedef void (*RedisModuleTypeAuxSaveFunc)(RedisModuleIO *rdb, int when); typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value); typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value); typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value); @@ -174,7 +180,7 @@ typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const cha typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); -#define REDISMODULE_TYPE_METHOD_VERSION 1 +#define REDISMODULE_TYPE_METHOD_VERSION 2 typedef struct RedisModuleTypeMethods { uint64_t version; RedisModuleTypeLoadFunc rdb_load; @@ -183,6 +189,9 @@ typedef struct RedisModuleTypeMethods { RedisModuleTypeMemUsageFunc mem_usage; RedisModuleTypeDigestFunc digest; RedisModuleTypeFreeFunc free; + RedisModuleTypeAuxLoadFunc aux_load; + RedisModuleTypeAuxSaveFunc aux_save; + int aux_save_triggers; } RedisModuleTypeMethods; #define REDISMODULE_GET_API(name) \ diff --git a/src/server.h b/src/server.h index b200a6696..33522dd69 100644 --- a/src/server.h +++ b/src/server.h @@ -536,6 +536,10 @@ typedef long long mstime_t; /* millisecond time type. */ #define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK) #define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS) +/* Bit flags for moduleTypeAuxSaveFunc */ +#define REDISMODULE_AUX_BEFORE_RDB (1<<0) +#define REDISMODULE_AUX_AFTER_RDB (1<<1) + struct RedisModule; struct RedisModuleIO; struct RedisModuleDigest; @@ -548,6 +552,8 @@ struct redisObject; * is deleted. */ typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver); typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value); +typedef int (*moduleTypeAuxLoadFunc)(struct RedisModuleIO *rdb, int encver, int when); +typedef void (*moduleTypeAuxSaveFunc)(struct RedisModuleIO *rdb, int when); typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value); typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value); typedef size_t (*moduleTypeMemUsageFunc)(const void *value); @@ -564,6 +570,9 @@ typedef struct RedisModuleType { moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; + moduleTypeAuxLoadFunc aux_load; + moduleTypeAuxSaveFunc aux_save; + int aux_save_triggers; char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */ } moduleType; @@ -1528,6 +1537,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +ssize_t rdbSaveModulesAux(rio *rdb, int when); /* Utils */ long long ustime(void); diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20afa..6e4574747 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,16 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so testrdb.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +testrdb.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +testrdb.so: testrdb.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c new file mode 100644 index 000000000..415497a2f --- /dev/null +++ b/tests/modules/testrdb.c @@ -0,0 +1,229 @@ +#include "redismodule.h" + +#include +#include + +/* Module configuration, save aux or not? */ +long long conf_aux_count = 0; + +/* Registered type */ +RedisModuleType *testrdb_type = NULL; + +/* Global values to store and persist to aux */ +RedisModuleString *before_str = NULL; +RedisModuleString *after_str = NULL; + +void *testrdb_type_load(RedisModuleIO *rdb, int encver) { + int count = RedisModule_LoadSigned(rdb); + assert(count==1); + assert(encver==1); + RedisModuleString *str = RedisModule_LoadString(rdb); + return str; +} + +void testrdb_type_save(RedisModuleIO *rdb, void *value) { + RedisModuleString *str = (RedisModuleString*)value; + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, str); +} + +void testrdb_aux_save(RedisModuleIO *rdb, int when) { + if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB); + if (conf_aux_count==0) assert(0); + if (when == REDISMODULE_AUX_BEFORE_RDB) { + if (before_str) { + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, before_str); + } else { + RedisModule_SaveSigned(rdb, 0); + } + } else { + if (after_str) { + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, after_str); + } else { + RedisModule_SaveSigned(rdb, 0); + } + } +} + +int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { + assert(encver == 1); + if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB); + if (conf_aux_count==0) assert(0); + RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb); + if (when == REDISMODULE_AUX_BEFORE_RDB) { + if (before_str) + RedisModule_FreeString(ctx, before_str); + before_str = NULL; + int count = RedisModule_LoadSigned(rdb); + if (count) + before_str = RedisModule_LoadString(rdb); + } else { + if (after_str) + RedisModule_FreeString(ctx, after_str); + after_str = NULL; + int count = RedisModule_LoadSigned(rdb); + if (count) + after_str = RedisModule_LoadString(rdb); + } + return REDISMODULE_OK; +} + +void testrdb_type_free(void *value) { + RedisModule_FreeString(NULL, (RedisModuleString*)value); +} + +int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + if (before_str) + RedisModule_FreeString(ctx, before_str); + before_str = argv[1]; + RedisModule_RetainString(ctx, argv[1]); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + if (argc != 1){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + if (before_str) + RedisModule_ReplyWithString(ctx, before_str); + else + RedisModule_ReplyWithStringBuffer(ctx, "", 0); + return REDISMODULE_OK; +} + +int testrdb_set_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + if (after_str) + RedisModule_FreeString(ctx, after_str); + after_str = argv[1]; + RedisModule_RetainString(ctx, argv[1]); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + if (argc != 1){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + if (after_str) + RedisModule_ReplyWithString(ctx, after_str); + else + RedisModule_ReplyWithStringBuffer(ctx, "", 0); + return REDISMODULE_OK; +} + +int testrdb_set_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 3){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + RedisModuleString *str = RedisModule_ModuleTypeGetValue(key); + if (str) + RedisModule_FreeString(ctx, str); + RedisModule_ModuleTypeSetValue(key, testrdb_type, argv[2]); + RedisModule_RetainString(ctx, argv[2]); + RedisModule_CloseKey(key); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + RedisModuleString *str = RedisModule_ModuleTypeGetValue(key); + RedisModule_CloseKey(key); + RedisModule_ReplyWithString(ctx, str); + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (argc > 0) + RedisModule_StringToLongLong(argv[0], &conf_aux_count); + + if (conf_aux_count==0) { + RedisModuleTypeMethods datatype_methods = { + .version = 1, + .rdb_load = testrdb_type_load, + .rdb_save = testrdb_type_save, + .aof_rewrite = NULL, + .digest = NULL, + .free = testrdb_type_free, + }; + + testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods); + if (testrdb_type == NULL) + return REDISMODULE_ERR; + } else { + RedisModuleTypeMethods datatype_methods = { + .version = REDISMODULE_TYPE_METHOD_VERSION, + .rdb_load = testrdb_type_load, + .rdb_save = testrdb_type_save, + .aof_rewrite = NULL, + .digest = NULL, + .free = testrdb_type_free, + .aux_load = testrdb_aux_load, + .aux_save = testrdb_aux_save, + .aux_save_triggers = (conf_aux_count == 1 ? + REDISMODULE_AUX_AFTER_RDB : + REDISMODULE_AUX_BEFORE_RDB | REDISMODULE_AUX_AFTER_RDB) + }; + + testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods); + if (testrdb_type == NULL) + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx,"testrdb.set.before", testrdb_set_before,"deny-oom",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.before", testrdb_get_before,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.set.after", testrdb_set_after,"deny-oom",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.after", testrdb_get_after,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.set.key", testrdb_set_key,"deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.key", testrdb_get_key,"",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl new file mode 100644 index 000000000..22201a08e --- /dev/null +++ b/tests/unit/moduleapi/testrdb.tcl @@ -0,0 +1,62 @@ +set testmodule [file normalize tests/modules/testrdb.so] + +proc restart_and_wait {} { + catch { + r debug restart + } + + # wait for the server to come back up + set retry 50 + while {$retry} { + if {[catch { r ping }]} { + after 100 + } else { + break + } + incr retry -1 + } +} + +tags "modules" { + start_server [list overrides [list loadmodule "$testmodule"]] { + test {modules are able to persist types} { + r testrdb.set.key key1 value1 + assert_equal "value1" [r testrdb.get.key key1] + r debug reload + assert_equal "value1" [r testrdb.get.key key1] + } + + test {modules global are lost without aux} { + r testrdb.set.before global1 + assert_equal "global1" [r testrdb.get.before] + restart_and_wait + assert_equal "" [r testrdb.get.before] + } + } + + start_server [list overrides [list loadmodule "$testmodule 2"]] { + test {modules are able to persist globals before and after} { + r testrdb.set.before global1 + r testrdb.set.after global2 + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + restart_and_wait + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + } + + } + + start_server [list overrides [list loadmodule "$testmodule 1"]] { + test {modules are able to persist globals just after} { + r testrdb.set.after global2 + assert_equal "global2" [r testrdb.get.after] + restart_and_wait + assert_equal "global2" [r testrdb.get.after] + } + } + + + # TODO: test short read handling + +} From dedbc9d4d24146aec715b22261331aaae26e891e Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sat, 6 Jul 2019 18:32:58 -0700 Subject: [PATCH 015/117] Removed unecessary creation of Redis objects --- src/cluster.c | 13 +++---------- src/config.c | 6 ++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index c85e3791d..e22222700 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4251,12 +4251,7 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - robj *o; - sds ci = clusterGenNodesDescription(0); - - o = createObject(OBJ_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); + addReplyBulkSds(c,clusterGenNodesDescription(0)); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4832,7 +4827,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) { * DUMP is actually not used by Redis Cluster but it is the obvious * complement of RESTORE and can be useful for different applications. */ void dumpCommand(client *c) { - robj *o, *dumpobj; + robj *o; rio payload; /* Check if the key is here. */ @@ -4845,9 +4840,7 @@ void dumpCommand(client *c) { createDumpPayload(&payload,o,c->argv[1]); /* Transfer to the client */ - dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); + addReplyBulkSds(c,payload.io.buffer.ptr); return; } diff --git a/src/config.c b/src/config.c index 7f0e9af89..ddf31d168 100644 --- a/src/config.c +++ b/src/config.c @@ -1602,12 +1602,10 @@ void configGetCommand(client *c) { matches++; } if (stringmatch(pattern,"notify-keyspace-events",1)) { - robj *flagsobj = createObject(OBJ_STRING, - keyspaceEventsFlagsToString(server.notify_keyspace_events)); + sds flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); addReplyBulkCString(c,"notify-keyspace-events"); - addReplyBulk(c,flagsobj); - decrRefCount(flagsobj); + addReplyBulkSds(c,flags); matches++; } if (stringmatch(pattern,"bind",1)) { From 4520e878002d08339b389fa33794999ee53061fb Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 19 Jul 2019 10:22:40 -0700 Subject: [PATCH 016/117] Hide HELLO and AUTH from slowlog and monitor --- src/server.c | 13 +++++++++---- src/server.h | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/server.c b/src/server.c index 2643d7266..0b79f6a62 100644 --- a/src/server.c +++ b/src/server.c @@ -146,6 +146,8 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */ * in this condition but just a few. * * no-monitor: Do not automatically propagate the command on MONITOR. + * + * no-slowlog: Do not automatically propagate the command to the slowlog. * * cluster-asking: Perform an implicit ASKING for this command, so the * command will be accepted in cluster mode if the slot is marked @@ -627,7 +629,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"auth",authCommand,-2, - "no-script ok-loading ok-stale fast @connection", + "no-script ok-loading ok-stale fast no-monitor no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* We don't allow PING during loading since in Redis PING is used as @@ -670,7 +672,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"exec",execCommand,1, - "no-script no-monitor @transaction", + "no-script no-monitor no-slowlog @transaction", 0,NULL,0,0,0,0,0,0}, {"discard",discardCommand,1, @@ -822,7 +824,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"hello",helloCommand,-2, - "no-script fast @connection", + "no-script fast no-monitor no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* EVAL can modify the dataset, however it is not flagged as a write @@ -2902,6 +2904,8 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) { c->flags |= CMD_STALE; } else if (!strcasecmp(flag,"no-monitor")) { c->flags |= CMD_SKIP_MONITOR; + } else if (!strcasecmp(flag,"no-slowlog")) { + c->flags |= CMD_SKIP_SLOWLOG; } else if (!strcasecmp(flag,"cluster-asking")) { c->flags |= CMD_ASKING; } else if (!strcasecmp(flag,"fast")) { @@ -3186,7 +3190,7 @@ void call(client *c, int flags) { /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ - if (flags & CMD_CALL_SLOWLOG && c->cmd->proc != execCommand) { + if (flags & CMD_CALL_SLOWLOG && !(c->flags & CMD_SKIP_SLOWLOG)) { char *latency_event = (c->cmd->flags & CMD_FAST) ? "fast-command" : "command"; latencyAddSampleIfNeeded(latency_event,duration/1000); @@ -3677,6 +3681,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) { flagcount += addReplyCommandFlag(c,cmd,CMD_LOADING, "loading"); flagcount += addReplyCommandFlag(c,cmd,CMD_STALE, "stale"); flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_MONITOR, "skip_monitor"); + flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog"); flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking"); flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast"); if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) || diff --git a/src/server.h b/src/server.h index 3987ab5fc..ecb54a3cf 100644 --- a/src/server.h +++ b/src/server.h @@ -218,35 +218,36 @@ typedef long long mstime_t; /* millisecond time type. */ #define CMD_LOADING (1ULL<<9) /* "ok-loading" flag */ #define CMD_STALE (1ULL<<10) /* "ok-stale" flag */ #define CMD_SKIP_MONITOR (1ULL<<11) /* "no-monitor" flag */ -#define CMD_ASKING (1ULL<<12) /* "cluster-asking" flag */ -#define CMD_FAST (1ULL<<13) /* "fast" flag */ +#define CMD_SKIP_SLOWLOG (1ULL<<12) /* "no-slowlog" flag */ +#define CMD_ASKING (1ULL<<13) /* "cluster-asking" flag */ +#define CMD_FAST (1ULL<<14) /* "fast" flag */ /* Command flags used by the module system. */ -#define CMD_MODULE_GETKEYS (1ULL<<14) /* Use the modules getkeys interface. */ -#define CMD_MODULE_NO_CLUSTER (1ULL<<15) /* Deny on Redis Cluster. */ +#define CMD_MODULE_GETKEYS (1ULL<<15) /* Use the modules getkeys interface. */ +#define CMD_MODULE_NO_CLUSTER (1ULL<<16) /* Deny on Redis Cluster. */ /* Command flags that describe ACLs categories. */ -#define CMD_CATEGORY_KEYSPACE (1ULL<<16) -#define CMD_CATEGORY_READ (1ULL<<17) -#define CMD_CATEGORY_WRITE (1ULL<<18) -#define CMD_CATEGORY_SET (1ULL<<19) -#define CMD_CATEGORY_SORTEDSET (1ULL<<20) -#define CMD_CATEGORY_LIST (1ULL<<21) -#define CMD_CATEGORY_HASH (1ULL<<22) -#define CMD_CATEGORY_STRING (1ULL<<23) -#define CMD_CATEGORY_BITMAP (1ULL<<24) -#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<25) -#define CMD_CATEGORY_GEO (1ULL<<26) -#define CMD_CATEGORY_STREAM (1ULL<<27) -#define CMD_CATEGORY_PUBSUB (1ULL<<28) -#define CMD_CATEGORY_ADMIN (1ULL<<29) -#define CMD_CATEGORY_FAST (1ULL<<30) -#define CMD_CATEGORY_SLOW (1ULL<<31) -#define CMD_CATEGORY_BLOCKING (1ULL<<32) -#define CMD_CATEGORY_DANGEROUS (1ULL<<33) -#define CMD_CATEGORY_CONNECTION (1ULL<<34) -#define CMD_CATEGORY_TRANSACTION (1ULL<<35) -#define CMD_CATEGORY_SCRIPTING (1ULL<<36) +#define CMD_CATEGORY_KEYSPACE (1ULL<<17) +#define CMD_CATEGORY_READ (1ULL<<18) +#define CMD_CATEGORY_WRITE (1ULL<<19) +#define CMD_CATEGORY_SET (1ULL<<20) +#define CMD_CATEGORY_SORTEDSET (1ULL<<21) +#define CMD_CATEGORY_LIST (1ULL<<22) +#define CMD_CATEGORY_HASH (1ULL<<23) +#define CMD_CATEGORY_STRING (1ULL<<24) +#define CMD_CATEGORY_BITMAP (1ULL<<25) +#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<26) +#define CMD_CATEGORY_GEO (1ULL<<27) +#define CMD_CATEGORY_STREAM (1ULL<<28) +#define CMD_CATEGORY_PUBSUB (1ULL<<29) +#define CMD_CATEGORY_ADMIN (1ULL<<30) +#define CMD_CATEGORY_FAST (1ULL<<31) +#define CMD_CATEGORY_SLOW (1ULL<<32) +#define CMD_CATEGORY_BLOCKING (1ULL<<33) +#define CMD_CATEGORY_DANGEROUS (1ULL<<34) +#define CMD_CATEGORY_CONNECTION (1ULL<<35) +#define CMD_CATEGORY_TRANSACTION (1ULL<<36) +#define CMD_CATEGORY_SCRIPTING (1ULL<<37) /* AOF states */ #define AOF_OFF 0 /* AOF is off */ From 408ca31d16e839a47807269e3886e962e12947b8 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 23 Jul 2019 15:25:00 +0800 Subject: [PATCH 017/117] Client side caching: do not reclaim tracking table if it's empty --- src/tracking.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 1189e82d5..badec69a7 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -206,8 +206,6 @@ void trackingInvalidateKey(robj *keyobj) { * flush the table: it will slowly get garbage collected as more keys * are modified in the used caching slots. */ void trackingInvalidateKeysOnFlush(int dbid) { - UNUSED(dbid); - if (server.tracking_clients) { listNode *ln; listIter li; @@ -222,12 +220,11 @@ void trackingInvalidateKeysOnFlush(int dbid) { /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { - for (int j = 0; j < TRACKING_TABLE_SIZE; j++) { + for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) { if (TrackingTable[j] != NULL) { raxFree(TrackingTable[j]); TrackingTable[j] = NULL; TrackingTableUsedSlots--; - if (TrackingTableUsedSlots == 0) break; } } From 79a73bb95a0c59bce292f8aba8e2b59a0851c3f3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Jul 2019 10:57:22 +0200 Subject: [PATCH 018/117] Client side caching: implement full slot limit function. --- src/server.c | 3 +++ src/server.h | 2 ++ src/tracking.c | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/server.c b/src/server.c index 5e18b5465..1f502e4ff 100644 --- a/src/server.c +++ b/src/server.c @@ -2403,6 +2403,9 @@ void initServerConfig(void) { /* Latency monitor */ server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; + /* Tracking. */ + server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_MAX_FILL; + /* Debugging */ server.assert_failed = ""; server.assert_file = ""; diff --git a/src/server.h b/src/server.h index fdab53830..172b1cf64 100644 --- a/src/server.h +++ b/src/server.h @@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ +#define CONFIG_DEFAULT_TRACKING_MAX_FILL 10 /* 10% tracking table max fill. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ @@ -1316,6 +1317,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ + int tracking_table_max_fill; /* Max fill percentage. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; diff --git a/src/tracking.c b/src/tracking.c index badec69a7..1a5fbe47c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -251,4 +251,40 @@ void trackingInvalidateKeysOnFlush(int dbid) { * random caching slots, and send invalidation messages to clients like if * the key was modified. */ void trackingLimitUsedSlots(void) { + static unsigned int timeout_counter = 0; + + if (server.tracking_table_max_fill == 0) return; /* No limits set. */ + unsigned int max_slots = + (TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill; + if (TrackingTableUsedSlots <= max_slots) { + timeout_counter = 0; + return; /* Limit not reached. */ + } + + /* We have to invalidate a few slots to reach the limit again. The effort + * we do here is proportional to the number of times we entered this + * function and found that we are still over the limit. */ + int effort = 100 * (timeout_counter+1); + + /* Let's start at a random position, and perform linear probing, in order + * to improve cache locality. However once we are able to find an used + * slot, jump again randomly, in order to avoid creating big holes in the + * table (that will make this funciton use more resourced later). */ + while(effort > 0) { + unsigned int idx = rand() % TRACKING_TABLE_SIZE; + do { + effort--; + idx = (idx+1) % TRACKING_TABLE_SIZE; + if (TrackingTable[idx] != NULL) { + trackingInvalidateSlot(idx); + if (TrackingTableUsedSlots <= max_slots) { + timeout_counter = 0; + return; /* Return ASAP: we are again under the limit. */ + } else { + break; /* Jump to next random position. */ + } + } + } while(effort > 0); + } + timeout_counter++; } From 1d54931685f8e9137d04112b45f51c25c0cd30b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Jul 2019 11:02:14 +0200 Subject: [PATCH 019/117] Client side caching: show tracking slots usage in INFO. --- src/server.c | 6 ++++-- src/server.h | 1 + src/tracking.c | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 1f502e4ff..0acf8545c 100644 --- a/src/server.c +++ b/src/server.c @@ -4147,7 +4147,8 @@ sds genRedisInfoString(char *section) { "active_defrag_hits:%lld\r\n" "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" - "active_defrag_key_misses:%lld\r\n", + "active_defrag_key_misses:%lld\r\n" + "tracking_used_slots:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4173,7 +4174,8 @@ sds genRedisInfoString(char *section) { server.stat_active_defrag_hits, server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, - server.stat_active_defrag_key_misses); + server.stat_active_defrag_key_misses, + trackingGetUsedSlots()); } /* Replication */ diff --git a/src/server.h b/src/server.h index 172b1cf64..e363f03f4 100644 --- a/src/server.h +++ b/src/server.h @@ -1642,6 +1642,7 @@ void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); +unsigned long long trackingGetUsedSlots(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index 1a5fbe47c..f7f0fc755 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -288,3 +288,9 @@ void trackingLimitUsedSlots(void) { } timeout_counter++; } + +/* This is just used in order to access the amount of used slots in the + * tracking table. */ +unsigned long long trackingGetUsedSlots(void) { + return TrackingTableUsedSlots; +} From 61fe76a7b9c3b05f3062d479601ce7bb2ac3577d Mon Sep 17 00:00:00 2001 From: wubostc <913721086@qq.com> Date: Wed, 24 Jul 2019 16:22:26 +0800 Subject: [PATCH 020/117] Reduce the calling stack --- src/t_zset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index fb7078abd..2680e76a9 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1357,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { /* Optimize: check if the element is too large or the list * becomes too long *before* executing zzlInsert. */ zobj->ptr = zzlInsert(zobj->ptr,ele,score); - if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries) - zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); - if (sdslen(ele) > server.zset_max_ziplist_value) + if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries || + sdslen(ele) > server.zset_max_ziplist_value) zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); if (newscore) *newscore = score; *flags |= ZADD_ADDED; From 1ead7a115817e60d61a2ffe4c774c46273f2787d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:33:53 +0200 Subject: [PATCH 021/117] Remove experimental warning from diskless replication. --- redis.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/redis.conf b/redis.conf index 74b6c018f..a3c98341f 100644 --- a/redis.conf +++ b/redis.conf @@ -336,10 +336,6 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- -# # New replicas and reconnecting replicas that are not able to continue the replication # process just receiving differences, need to do what is called a "full # synchronization". An RDB file is transmitted from the master to the replicas. From 83b8fbf58020ac26045dbc32c7332e960960b33a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:35:01 +0200 Subject: [PATCH 022/117] Client side caching: config option for table fill rate. --- src/config.c | 18 ++++++++++++++++-- src/server.c | 2 +- src/server.h | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fde00ddf5..7ffc94b35 100644 --- a/src/config.c +++ b/src/config.c @@ -686,6 +686,17 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { server.slowlog_max_len = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"tracking-table-max-fill") && + argc == 2) + { + server.tracking_table_max_fill = strtoll(argv[1],NULL,10); + if (server.tracking_table_max_fill > 100 || + server.tracking_table_max_fill < 0) + { + err = "The tracking table fill percentage must be an " + "integer between 0 and 100"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -1133,6 +1144,8 @@ void configSetCommand(client *c) { "slowlog-max-len",ll,0,LONG_MAX) { /* Cast to unsigned. */ server.slowlog_max_len = (unsigned long)ll; + } config_set_numerical_field( + "tracking-table-max-fill",server.tracking_table_max_fill,0,100) { } config_set_numerical_field( "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ } config_set_numerical_field( @@ -1338,8 +1351,8 @@ void configGetCommand(client *c) { server.slowlog_log_slower_than); config_get_numerical_field("latency-monitor-threshold", server.latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", - server.slowlog_max_len); + config_get_numerical_field("slowlog-max-len", server.slowlog_max_len); + config_get_numerical_field("tracking-table-max-fill", server.tracking_table_max_fill); config_get_numerical_field("port",server.port); config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); @@ -2167,6 +2180,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); + rewriteConfigNumericalOption(state,"tracking-table-max-fill",server.tracking_table_max_fill,CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); diff --git a/src/server.c b/src/server.c index 0acf8545c..eb5cef386 100644 --- a/src/server.c +++ b/src/server.c @@ -2404,7 +2404,7 @@ void initServerConfig(void) { server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; /* Tracking. */ - server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_MAX_FILL; + server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL; /* Debugging */ server.assert_failed = ""; diff --git a/src/server.h b/src/server.h index e363f03f4..399a0bbb6 100644 --- a/src/server.h +++ b/src/server.h @@ -171,7 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ -#define CONFIG_DEFAULT_TRACKING_MAX_FILL 10 /* 10% tracking table max fill. */ +#define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ From 27095db3f3b7473e788f144b6ce666a6bbd8f5e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:38:11 +0200 Subject: [PATCH 023/117] Example redis.conf: stay under 80 cols. --- redis.conf | 64 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/redis.conf b/redis.conf index a3c98341f..28a9889ea 100644 --- a/redis.conf +++ b/redis.conf @@ -336,9 +336,11 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. +# # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB @@ -348,14 +350,14 @@ replica-read-only yes # RDB file to replica sockets, without touching the disk at all. # # With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -366,8 +368,8 @@ repl-diskless-sync no # to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. @@ -389,9 +391,9 @@ repl-diskless-sync-delay 5 # sufficient memory, if you don't have it, you risk an OOM kill. repl-diskless-load disabled -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -423,10 +425,10 @@ repl-diskless-load disabled repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # # The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. @@ -448,13 +450,13 @@ repl-disable-tcp-nodelay no # # repl-backlog-ttl 3600 -# The replica priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # # A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the replica as not able to perform the # role of master, so a replica with priority of 0 will never be selected by @@ -743,17 +745,17 @@ replica-priority 100 # DEL commands to the replica as keys evict in the master side. # # This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). # # Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. # # replica-ignore-maxmemory yes From 63e1d1cb27bf4106cc4684e18800fb9f2aa5fb2f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:45:10 +0200 Subject: [PATCH 024/117] Client side caching: document tracking-table-max-fill in redis.conf. --- redis.conf | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/redis.conf b/redis.conf index 28a9889ea..c242609ec 100644 --- a/redis.conf +++ b/redis.conf @@ -516,6 +516,39 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# 16 millions of slots, what clients may have certain subsets of keys. In turn +# this is used in order to send invalidation messages to clients. Please +# to understand more about the feature check this page: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 10%, and once this limit is +# reached, Redis will start to evict caching slots in the invalidation table +# even if keys are not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum fill rate is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and all the 16 +# millions of caching slots can be used at the same time. In the "stats" +# INFO section, you can find information about the amount of caching slots +# used at every given moment. +# +# tracking-table-max-fill 10 + ################################## SECURITY ################################### # Warning: since Redis is pretty fast an outside user can try up to From 4e547a0ac5ab577b1abe5935137f56a031c2e06f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 25 Jul 2019 19:17:58 +0200 Subject: [PATCH 025/117] Mark diskless loads as experimental in redis.conf. --- redis.conf | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/redis.conf b/redis.conf index c242609ec..50ba823ac 100644 --- a/redis.conf +++ b/redis.conf @@ -375,15 +375,25 @@ repl-diskless-sync no # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replica can load the rdb it reads from the replication link directly from the -# socket, or store the rdb to a file and read that file after it was completely +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if your do what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely # recived from the master. +# # In many cases the disk is slower than the network, and storing and loading -# the rdb file may increase replication time (and even increase the master's +# the RDB file may increase replication time (and even increase the master's # Copy on Write memory and salve buffers). -# However, parsing the rdb file directly from the socket may mean that we have -# to flush the contents of the current database before the full rdb was received. -# for this reason we have the following options: +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# # "disabled" - Don't use diskless load (store the rdb file to the disk first) # "on-empty-db" - Use diskless load only when it is completely safe. # "swapdb" - Keep a copy of the current db contents in RAM while parsing From 920cbf051a2229f4f5f3e45812275d6eae8d3141 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 28 Jul 2019 14:33:57 +0100 Subject: [PATCH 026/117] Updating resident memory request impl on FreeBSD. --- src/zmalloc.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index 5e6010278..fd8bb6938 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -294,6 +294,26 @@ size_t zmalloc_get_rss(void) { return t_info.resident_size; } +#elif defined(__FreeBSD__) +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + struct kinfo_proc info; + size_t infolen = sizeof(info); + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = getpid(); + + if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0) + return (size_t)info.ki_rssize; + + return 0L; +} #else size_t zmalloc_get_rss(void) { /* If we can't get the RSS in an OS-specific way for this system just From c6d5347a22079c3d93d02b59a63262395e69f92c Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 29 Jul 2019 18:11:52 -0400 Subject: [PATCH 027/117] Fix HLL corruption bug --- src/hyperloglog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index e01ea6042..e0557f985 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -701,6 +701,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { first += span; } if (span == 0) return -1; /* Invalid format. */ + if (span >= end) return -1; /* Invalid format. */ next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; if (next >= end) next = NULL; From fe6d9e4c954753ef7dadc17510fee95b69edcca3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 30 Jul 2019 11:20:51 +0200 Subject: [PATCH 028/117] emptyDbGeneric(): call signalFlushDb() before deleting the keys. This was broken since a refactoring performed recently by myself. --- src/db.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 3d8b844e8..95eaf04e9 100644 --- a/src/db.c +++ b/src/db.c @@ -353,6 +353,11 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + int startdb, enddb; if (dbnum == -1) { startdb = 0; @@ -378,7 +383,6 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( } } if (dbnum == -1) flushSlaveKeysWithExpireList(); - signalFlushedDb(dbnum); return removed; } From 9fa285523c2070d59e0b992ecf43bb98f8b5c448 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 15:11:57 +0300 Subject: [PATCH 029/117] Avoid diskelss-load if modules did not declare they handle read errors --- src/module.c | 47 +++++++++++++++++++++++++++++++++++------------ src/redismodule.h | 10 ++++------ src/replication.c | 6 +++++- src/server.h | 3 +-- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/module.c b/src/module.c index 68952b1e0..36384c018 100644 --- a/src/module.c +++ b/src/module.c @@ -51,6 +51,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int options; /* Moduile options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -771,6 +772,19 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Set flags defining capabilities or behavior bit flags. + * + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: + * Generally, modules don't need to bother with this, as the process will just + * terminate if a read error happens, however, setting this flag would allow + * repl-diskless-load to work if enabled. + * The module should use RedisModule_IsIOError after reads, before using the + * data that was read, and in case of error, propagate it upwards, and also be + * able to release the partially populated value and all it's allocations. */ +void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { + ctx->module->options = options; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -3143,7 +3157,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } @@ -3157,19 +3171,29 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } -/* Set flags defining capabilities or behavior */ -void RM_SetIOFlags(RedisModuleIO *io, int flags) { - io->flags = flags; -} +/* Returns 0 if there's at least one registered data type that did not declare + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should + * be avoided since it could cause data loss. */ +int moduleAllDatatypesHandleErrors() { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; -/* Get flags which were set by RedisModule_SetIOFlags */ -int RM_GetIOFlags(RedisModuleIO *io) { - return io->flags; + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (listLength(module->types) && + !(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)) + { + dictReleaseIterator(di); + return 0; + } + } + dictReleaseIterator(di); + return 1; } /* Returns true if any previous IO API failed. - * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with - * RediModule_SetIOFlags first. */ + * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetModuleOptions first. */ int RM_IsIOError(RedisModuleIO *io) { return io->error; } @@ -5434,9 +5458,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); - REGISTER_API(GetIOFlags); - REGISTER_API(SetIOFlags); REGISTER_API(IsIOError); + REGISTER_API(SetModuleOptions); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index 154b5b405..f0f27c067 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,8 +140,8 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) -/* Declare that the module can handle errors with RedisModule_SetIOFlags */ -#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) +/* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ +#define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) /* ------------------------- End of common defines ------------------------ */ @@ -274,9 +274,8 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); -void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); -int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -450,9 +449,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); - REDISMODULE_GET_API(SetIOFlags); - REDISMODULE_GET_API(GetIOFlags); REDISMODULE_GET_API(IsIOError); + REDISMODULE_GET_API(SetModuleOptions); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..14ae2f96c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1115,8 +1115,12 @@ void restartAOFAfterSYNC() { static int useDisklessLoad() { /* compute boolean decision to use diskless load */ - return server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || + int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); + /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ + if (enabled && !moduleAllDatatypesHandleErrors()) + enabled = 0; + return enabled; } /* Helper function for readSyncBulkPayload() to make backups of the current diff --git a/src/server.h b/src/server.h index 9957c3b5c..5991cfa6c 100644 --- a/src/server.h +++ b/src/server.h @@ -599,7 +599,6 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ - int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -612,7 +611,6 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ - iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis @@ -1530,6 +1528,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +int moduleAllDatatypesHandleErrors(); /* Utils */ long long ustime(void); From 037ac27e41004deaca686f206b01cd4799ce493b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 21 Jul 2019 18:18:11 +0300 Subject: [PATCH 030/117] Add test for module diskless short reads --- tests/modules/testrdb.c | 13 ++++++- tests/unit/moduleapi/testrdb.tcl | 62 +++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index 415497a2f..d73c8bfd3 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -15,6 +15,8 @@ RedisModuleString *after_str = NULL; void *testrdb_type_load(RedisModuleIO *rdb, int encver) { int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return NULL; assert(count==1); assert(encver==1); RedisModuleString *str = RedisModule_LoadString(rdb); @@ -57,6 +59,8 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, before_str); before_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) before_str = RedisModule_LoadString(rdb); } else { @@ -64,14 +68,19 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, after_str); after_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) after_str = RedisModule_LoadString(rdb); } + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; return REDISMODULE_OK; } void testrdb_type_free(void *value) { - RedisModule_FreeString(NULL, (RedisModuleString*)value); + if (value) + RedisModule_FreeString(NULL, (RedisModuleString*)value); } int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) @@ -171,6 +180,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; + RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS); + if (argc > 0) RedisModule_StringToLongLong(argv[0], &conf_aux_count); diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index 22201a08e..c72570002 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -56,7 +56,67 @@ tags "modules" { } } + tags {repl} { + test {diskless loading short read with module} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] - # TODO: test short read handling + # Set master and replica to use diskless replication + $master config set repl-diskless-sync yes + $master config set rdbcompression no + $replica config set repl-diskless-load swapdb + for {set k 0} {$k < 30} {incr k} { + r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]] + } + # Start the replication process... + $master config set repl-diskless-sync-delay 0 + $replica replicaof $master_host $master_port + + # kill the replication at various points + set attempts 3 + if {$::accurate} { set attempts 10 } + for {set i 0} {$i < $attempts} {incr i} { + # wait for the replica to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1 + + # add some additional random sleep so that we kill the master on a different place each time + after [expr {int(rand()*100)}] + + # kill the replica connection on the master + set killed [$master client kill type replica] + + if {[catch { + set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10] + if {$::verbose} { + puts $res + } + }]} { + puts "failed triggering short read" + # force the replica to try another full sync + $master client kill type replica + $master set asdf asdf + # the side effect of resizing the backlog is that it is flushed (16k is the min size) + $master config set repl-backlog-size [expr {16384 + $i}] + } + # wait for loading to stop (fail) + wait_for_condition 100 10 { + [s -1 loading] eq 0 + } else { + fail "Replica didn't disconnect" + } + } + # enable fast shutdown + $master config set rdb-key-save-delay 0 + } + } + } + } } From 4b9c5d8536a4f5e11b0129cb32fc78913975be7d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 16:32:58 +0300 Subject: [PATCH 031/117] Log message when modules prevent diskless-load --- src/replication.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 14ae2f96c..614aaec83 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1118,8 +1118,11 @@ static int useDisklessLoad() { int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ - if (enabled && !moduleAllDatatypesHandleErrors()) + if (enabled && !moduleAllDatatypesHandleErrors()) { + serverLog(LL_WARNING, + "Skipping diskless-load because there are modules that don't handle read errors."); enabled = 0; + } return enabled; } From 6637823a4f899162141350dd58f201eb9854c3f2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 10:36:23 +0200 Subject: [PATCH 032/117] HyperLogLog: fix the fix of a corruption bug. --- src/hyperloglog.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index e0557f985..5d4afaa25 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -700,8 +700,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { p += oplen; first += span; } - if (span == 0) return -1; /* Invalid format. */ - if (span >= end) return -1; /* Invalid format. */ + if (span == 0 || p >= end) return -1; /* Invalid format. */ next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; if (next >= end) next = NULL; From 203a7762dcff800d73932907fd11968342d0d9ca Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 12:03:10 +0200 Subject: [PATCH 033/117] Make EMBSTR case of #6261 more obvious. --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index b9ff0a88b..fb011d311 100644 --- a/src/object.c +++ b/src/object.c @@ -472,7 +472,7 @@ robj *tryObjectEncoding(robj *o) { o->encoding = OBJ_ENCODING_INT; o->ptr = (void*) value; return o; - } else { + } else if (o->encoding == OBJ_ENCODING_EMBSTR) { decrRefCount(o); return createStringObjectFromLongLongForValue(value); } From b8f4ed24482d7a40fe96d49b1d47e9707c65a351 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 19:04:29 +0200 Subject: [PATCH 034/117] Fix regression causing EXEC to appear in the slow log. This was recently introduced with PR #6266. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 9de604315..5b3b434df 100644 --- a/src/server.c +++ b/src/server.c @@ -3200,7 +3200,7 @@ void call(client *c, int flags) { /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ - if (flags & CMD_CALL_SLOWLOG && !(c->flags & CMD_SKIP_SLOWLOG)) { + if (flags & CMD_CALL_SLOWLOG && !(c->cmd->flags & CMD_SKIP_SLOWLOG)) { char *latency_event = (c->cmd->flags & CMD_FAST) ? "fast-command" : "command"; latencyAddSampleIfNeeded(latency_event,duration/1000); From 89dc16f01d159551df27060f47d536f77f84bef4 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 2 Aug 2019 17:17:19 +0800 Subject: [PATCH 035/117] networking: flushSlavesOutputBuffers bugfix --- src/networking.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/networking.c b/src/networking.c index 7976caf29..a39fd73d8 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2468,7 +2468,6 @@ void flushSlavesOutputBuffers(void) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = listNodeValue(ln); - int events; /* Note that the following will not flush output buffers of slaves * in STATE_ONLINE but having put_online_on_ack set to true: in this @@ -2476,9 +2475,8 @@ void flushSlavesOutputBuffers(void) { * of put_online_on_ack is to postpone the moment it is installed. * This is what we want since slaves in this state should not receive * writes before the first ACK. */ - events = aeGetFileEvents(server.el,slave->fd); - if (events & AE_WRITABLE && - slave->replstate == SLAVE_STATE_ONLINE && + if (slave->replstate == SLAVE_STATE_ONLINE && + !slave->repl_put_online_on_ack && clientHasPendingReplies(slave)) { writeToClient(slave->fd,slave,0); From 766550a7c6ad6fba5e1ea99fcbfdfcd2f5496181 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 2 Aug 2019 20:13:16 +0200 Subject: [PATCH 036/117] tracking_collisions.c: initial skeleton. ... of a program to just test the hashing functions collisions on a 24 bit output with strings that are very likely Redis key names, and names of a kind that are particularly prone to collisions. --- utils/tracking_collisions.c | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 utils/tracking_collisions.c diff --git a/utils/tracking_collisions.c b/utils/tracking_collisions.c new file mode 100644 index 000000000..b996452e7 --- /dev/null +++ b/utils/tracking_collisions.c @@ -0,0 +1,40 @@ +/* This is a small program used in order to understand the collison rate + * of CRC64 (ISO version) VS other stronger hashing functions in the context + * of hashing keys for the Redis "tracking" feature (client side caching + * assisted by the server). + * + * The program attempts to hash keys with common names in the form of + * + * prefix: + * + * And counts the resulting collisons generated in the 24 bits of output + * needed for the tracking feature invalidation table (16 millions + entries) + * + * -------------------------------------------------------------------------- + * + * Copyright (C) 2019 Salvatore Sanfilippo + * This code is released under the BSD 2 clause license. + */ + +#include +#include +#include +#include + +#define TABLE_SIZE (1<<24) +int Table[TABLE_SIZE]; + +/* Test the hashing function provided as callback and return the + * number of collisions found. */ +unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { + unsigned long collisions = 0; + memset(Table,0,sizeof(Table)); + char *prefixes[] = {"object", "message", "user", NULL}; + return collisions; +} + +int main(void) { + printf("SHA1 : %lu\n", testHashingFunction(sha1Hash)); + printf("CRC64: %lu\n", testHashingFunction(crc64Hash)); + return 0; +} From 7cd02bbab9c34e09874b969d7d54d3232f077ba1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 2 Aug 2019 20:24:27 +0200 Subject: [PATCH 037/117] tracking_collisions.c: sha1 + crc64 implemented. --- utils/tracking_collisions.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/utils/tracking_collisions.c b/utils/tracking_collisions.c index b996452e7..cd64b36c5 100644 --- a/utils/tracking_collisions.c +++ b/utils/tracking_collisions.c @@ -10,6 +10,11 @@ * And counts the resulting collisons generated in the 24 bits of output * needed for the tracking feature invalidation table (16 millions + entries) * + * Compile with: + * + * cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c + * ./a.out + * * -------------------------------------------------------------------------- * * Copyright (C) 2019 Salvatore Sanfilippo @@ -20,16 +25,47 @@ #include #include #include +#include "../src/crc64.h" +#include "../src/sha1.h" #define TABLE_SIZE (1<<24) int Table[TABLE_SIZE]; +uint64_t crc64Hash(char *key, size_t len) { + return crc64(0,(unsigned char*)key,len); +} + +uint64_t sha1Hash(char *key, size_t len) { + SHA1_CTX ctx; + unsigned char hash[20]; + + SHA1Init(&ctx); + SHA1Update(&ctx,(unsigned char*)key,len); + SHA1Final(hash,&ctx); + uint64_t hash64; + memcpy(&hash64,hash,sizeof(hash64)); + return hash64; +} + /* Test the hashing function provided as callback and return the * number of collisions found. */ unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { unsigned long collisions = 0; memset(Table,0,sizeof(Table)); char *prefixes[] = {"object", "message", "user", NULL}; + for (int i = 0; prefixes[i] != NULL; i++) { + for (int j = 0; j < TABLE_SIZE/2; j++) { + char keyname[128]; + size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d", + prefixes[i],j); + uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE; + if (Table[bucket]) { + collisions++; + } else { + Table[bucket] = 1; + } + } + } return collisions; } From e87768e8ca3bf3d01e296bf88c2dd833689f7a4b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 5 Aug 2019 17:38:15 +0200 Subject: [PATCH 038/117] Replication: clarify why repl_put_online_on_ack exists at all. --- src/replication.c | 42 +++++++++++++++++++++++++++++++++--------- src/server.h | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..d6646c9ef 100644 --- a/src/replication.c +++ b/src/replication.c @@ -823,7 +823,9 @@ void replconfCommand(client *c) { c->repl_ack_time = server.unixtime; /* If this was a diskless replication, we need to really put * the slave online when the first ACK is received (which - * confirms slave is online and ready to get more data). */ + * confirms slave is online and ready to get more data). This + * allows for simpler and less CPU intensive EOF detection + * when streaming RDB files. */ if (c->repl_put_online_on_ack && c->replstate == SLAVE_STATE_ONLINE) putSlaveOnline(c); /* Note: this command does not reply anything! */ @@ -842,18 +844,20 @@ void replconfCommand(client *c) { addReply(c,shared.ok); } -/* This function puts a slave in the online state, and should be called just - * after a slave received the RDB file for the initial synchronization, and +/* This function puts a replica in the online state, and should be called just + * after a replica received the RDB file for the initial synchronization, and * we are finally ready to send the incremental stream of commands. * * It does a few things: * - * 1) Put the slave in ONLINE state (useless when the function is called - * because state is already ONLINE but repl_put_online_on_ack is true). + * 1) Put the slave in ONLINE state. Note that the function may also be called + * for a replicas that are already in ONLINE state, but having the flag + * repl_put_online_on_ack set to true: we still have to install the write + * handler in that case. This function will take care of that. * 2) Make sure the writable event is re-installed, since calling the SYNC * command disables it, so that we can accumulate output buffer without - * sending it to the slave. - * 3) Update the count of good slaves. */ + * sending it to the replica. + * 3) Update the count of "good replicas". */ void putSlaveOnline(client *slave) { slave->replstate = SLAVE_STATE_ONLINE; slave->repl_put_online_on_ack = 0; @@ -965,11 +969,31 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { serverLog(LL_NOTICE, "Streamed RDB transfer with replica %s succeeded (socket). Waiting for REPLCONF ACK from slave to enable streaming", replicationGetSlaveName(slave)); - /* Note: we wait for a REPLCONF ACK message from slave in + /* Note: we wait for a REPLCONF ACK message from the replica in * order to really put it online (install the write handler * so that the accumulated data can be transferred). However * we change the replication state ASAP, since our slave - * is technically online now. */ + * is technically online now. + * + * So things work like that: + * + * 1. We end trasnferring the RDB file via socket. + * 2. The replica is put ONLINE but the write handler + * is not installed. + * 3. The replica however goes really online, and pings us + * back via REPLCONF ACK commands. + * 4. Now we finally install the write handler, and send + * the buffers accumulated so far to the replica. + * + * But why we do that? Because the replica, when we stream + * the RDB directly via the socket, must detect the RDB + * EOF (end of file), that is a special random string at the + * end of the RDB (for streamed RDBs we don't know the length + * in advance). Detecting such final EOF string is much + * simpler and less CPU intensive if no more data is sent + * after such final EOF. So we don't want to glue the end of + * the RDB trasfer with the start of the other replication + * data. */ slave->replstate = SLAVE_STATE_ONLINE; slave->repl_put_online_on_ack = 1; slave->repl_ack_time = server.unixtime; /* Timeout otherwise. */ diff --git a/src/server.h b/src/server.h index 13e596836..d003ce59a 100644 --- a/src/server.h +++ b/src/server.h @@ -848,7 +848,7 @@ typedef struct client { uint64_t flags; /* Client flags: CLIENT_* macros. */ int authenticated; /* Needed when the default user requires auth. */ int replstate; /* Replication state if this is a slave. */ - int repl_put_online_on_ack; /* Install slave write handler on ACK. */ + int repl_put_online_on_ack; /* Install slave write handler on first ACK. */ int repldbfd; /* Replication DB file descriptor. */ off_t repldboff; /* Replication DB file offset. */ off_t repldbsize; /* Replication DB file size. */ From f8b7c18d345b59ef47a0c0ab1a0f172caa671fa3 Mon Sep 17 00:00:00 2001 From: Diego Bendersky Date: Mon, 5 Aug 2019 17:35:50 -0300 Subject: [PATCH 039/117] pass len to alloc in clusterManagerMigrateKeysInReply --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index e363a2795..904339857 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -3222,7 +3222,7 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source, redisReply *entry = reply->element[i]; size_t idx = i + offset; assert(entry->type == REDIS_REPLY_STRING); - argv[idx] = (char *) sdsnew(entry->str); + argv[idx] = (char *) sdsnewlen(entry->str, entry->len); argv_len[idx] = entry->len; if (dots) dots[i] = '.'; } From 5973b475cb8753b9f0019086e6470c1bce079ea7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 7 Aug 2019 13:05:33 +0300 Subject: [PATCH 040/117] RM_ReplyWithCString was missing registration --- src/module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.c b/src/module.c index 275ee4344..ab614c529 100644 --- a/src/module.c +++ b/src/module.c @@ -5389,6 +5389,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplyWithString); REGISTER_API(ReplyWithStringBuffer); + REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); From 8bf7c6d789986fd8a8823a1fb41eed8e498cbc59 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 25 Aug 2019 10:11:48 +0300 Subject: [PATCH 041/117] Modlue fork is killed when the parent exists --- src/module.c | 26 +++++++++++++++----------- src/server.c | 6 ++++++ src/server.h | 1 + 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/module.c b/src/module.c index 03cc36702..6f3be61af 100644 --- a/src/module.c +++ b/src/module.c @@ -5086,20 +5086,11 @@ int RM_ExitFromChild(int retcode) return REDISMODULE_OK; } -/* Can be used to kill the forked child process from the parent process. - * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ +void TerminateModuleForkChild(int wait) { int statloc; - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; - /* Kill module child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); - if (kill(server.module_child_pid,SIGUSR1) != -1) { + if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { while(wait3(&statloc,0,NULL) != server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ @@ -5108,6 +5099,19 @@ int RM_KillForkChild(int child_pid) moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + TerminateModuleForkChild(1); return REDISMODULE_OK; } diff --git a/src/server.c b/src/server.c index 675b638d6..31418e01b 100644 --- a/src/server.c +++ b/src/server.c @@ -3551,6 +3551,12 @@ int prepareForShutdown(int flags) { killRDBChild(); } + /* Kill module child if there is one. */ + if (server.module_child_pid != -1) { + serverLog(LL_WARNING,"There is a module fork child. Killing it!"); + TerminateModuleForkChild(0); + } + if (server.aof_state != AOF_OFF) { /* Kill the AOF saving child as the AOF we already have may be longer * but contains the full dataset anyway. */ diff --git a/src/server.h b/src/server.h index deaaf2636..7aa4bc2b7 100644 --- a/src/server.h +++ b/src/server.h @@ -1532,6 +1532,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); +void TerminateModuleForkChild(int wait); /* Utils */ long long ustime(void); From e6c2020a558988b6bd9c729f572eb4fe73e51b7a Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Sat, 31 Aug 2019 07:47:11 +0000 Subject: [PATCH 042/117] src/debug.c do not support aarch64 dump utcontext, so add the context of aarch64. The content comes from the definition of the sigcontext and tested on my aarch64 server. sigcontext defined at the linux kernel code: arch/arm64/include/uapi/asm/sigcontext.h --- src/debug.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..595e45f72 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1110,6 +1110,33 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.mc_cs ); logStackContent((void**)uc->uc_mcontext.mc_rsp); +#elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); #else serverLog(LL_WARNING, " Dumping of registers not supported for this OS/arch"); From 11228bd0fb1ca5fc743dba2342fbe7cdf409c779 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Aug 2019 14:40:09 +0200 Subject: [PATCH 043/117] Improve comment in flushSlavesOutputBuffers(). --- src/networking.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7976caf29..9ff2afd8d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2475,7 +2475,8 @@ void flushSlavesOutputBuffers(void) { * case the writable event is never installed, since the purpose * of put_online_on_ack is to postpone the moment it is installed. * This is what we want since slaves in this state should not receive - * writes before the first ACK. */ + * writes before the first ACK (to know the reason, grep for this + * flag in this file). */ events = aeGetFileEvents(server.el,slave->fd); if (events & AE_WRITABLE && slave->replstate == SLAVE_STATE_ONLINE && From 3b40a8858770286b217ebc1fdd97a6a9950bae46 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Aug 2019 14:46:21 +0200 Subject: [PATCH 044/117] More strict checks and better comments in flushSlaveOutputBuffers(). Related to #6296. --- src/networking.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/networking.c b/src/networking.c index 7bdd3c474..7555ca77d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2468,15 +2468,26 @@ void flushSlavesOutputBuffers(void) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = listNodeValue(ln); + int events = aeGetFileEvents(server.el,slave->fd); + int can_receive_writes = (events & AE_WRITABLE) || + (slave->flags & CLIENT_PENDING_WRITE); - /* Note that the following will not flush output buffers of slaves - * in STATE_ONLINE but having put_online_on_ack set to true: in this - * case the writable event is never installed, since the purpose - * of put_online_on_ack is to postpone the moment it is installed. - * This is what we want since slaves in this state should not receive - * writes before the first ACK (to know the reason, grep for this - * flag in this file). */ + /* We don't want to send the pending data to the replica in a few + * cases: + * + * 1. For some reason there is neither the write handler installed + * nor the client is flagged as to have pending writes: for some + * reason this replica may not be set to receive data. This is + * just for the sake of defensive programming. + * + * 2. The put_online_on_ack flag is true. To know why we don't want + * to send data to the replica in this case, please grep for the + * flag for this flag. + * + * 3. Obviously if the slave is not ONLINE. + */ if (slave->replstate == SLAVE_STATE_ONLINE && + can_receive_writes && !slave->repl_put_online_on_ack && clientHasPendingReplies(slave)) { From 4ff08c865715f1c379e9c0e119caaa3afcfa1ed6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Sep 2019 11:41:16 +0200 Subject: [PATCH 045/117] Cluster: abort loading nodes data if vars arguments are unbalanced. See for reference PR #6337. Thanks to @git-hulk for spotting this. --- src/cluster.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cluster.c b/src/cluster.c index e22222700..a2615fdc0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -138,6 +138,7 @@ int clusterLoadConfig(char *filename) { /* Handle the special "vars" line. Don't pretend it is the last * line even if it actually is when generated by Redis. */ if (strcasecmp(argv[0],"vars") == 0) { + if (!(argc % 2)) goto fmterr; for (j = 1; j < argc; j += 2) { if (strcasecmp(argv[j],"currentEpoch") == 0) { server.cluster->currentEpoch = From 2e1e170aad04f27d66551c95f1a2ef41d4930874 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Sep 2019 12:50:47 +0200 Subject: [PATCH 046/117] RESP3: fix cases of NULL reported instead of empty aggregate. --- src/server.c | 10 ++++++++++ src/server.h | 2 +- src/t_hash.c | 4 ++-- src/t_list.c | 4 ++-- src/t_set.c | 10 +++++----- src/t_zset.c | 18 +++++++++--------- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/server.c b/src/server.c index 5b3b434df..fb308e5f8 100644 --- a/src/server.c +++ b/src/server.c @@ -2161,6 +2161,16 @@ void createSharedObjects(void) { shared.nullarray[2] = createObject(OBJ_STRING,sdsnew("*-1\r\n")); shared.nullarray[3] = createObject(OBJ_STRING,sdsnew("_\r\n")); + shared.emptymap[0] = NULL; + shared.emptymap[1] = NULL; + shared.emptymap[2] = createObject(OBJ_STRING,sdsnew("*0\r\n")); + shared.emptymap[3] = createObject(OBJ_STRING,sdsnew("%0\r\n")); + + shared.emptyset[0] = NULL; + shared.emptyset[1] = NULL; + shared.emptyset[2] = createObject(OBJ_STRING,sdsnew("*0\r\n")); + shared.emptyset[3] = createObject(OBJ_STRING,sdsnew("~0\r\n")); + for (j = 0; j < PROTO_SHARED_SELECT_CMDS; j++) { char dictid_str[64]; int dictid_len; diff --git a/src/server.h b/src/server.h index d003ce59a..31d253066 100644 --- a/src/server.h +++ b/src/server.h @@ -897,7 +897,7 @@ struct moduleLoadQueueEntry { struct sharedObjectsStruct { robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space, - *colon, *queued, *null[4], *nullarray[4], + *colon, *queued, *null[4], *nullarray[4], *emptymap[4], *emptyset[4], *emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, diff --git a/src/t_hash.c b/src/t_hash.c index bc70e4051..e6ed33819 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -772,8 +772,8 @@ void genericHgetallCommand(client *c, int flags) { hashTypeIterator *hi; int length, count = 0; - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL - || checkType(c,o,OBJ_HASH)) return; + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymap[c->resp])) + == NULL || checkType(c,o,OBJ_HASH)) return; /* We return a map if the user requested keys and values, like in the * HGETALL case. Otherwise to use a flat array makes more sense. */ diff --git a/src/t_list.c b/src/t_list.c index 54e4959b9..601ea8899 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -402,7 +402,7 @@ void lrangeCommand(client *c) { if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL || checkType(c,o,OBJ_LIST)) return; llen = listTypeLength(o); @@ -414,7 +414,7 @@ void lrangeCommand(client *c) { /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } if (end >= llen) end = llen-1; diff --git a/src/t_set.c b/src/t_set.c index 05d9ee243..abbf82fde 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -418,10 +418,10 @@ void spopWithCountCommand(client *c) { if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; - /* If count is zero, serve an empty multibulk ASAP to avoid special + /* If count is zero, serve an empty set ASAP to avoid special * cases later. */ if (count == 0) { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); return; } @@ -632,13 +632,13 @@ void srandmemberWithCountCommand(client *c) { uniq = 0; } - if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) + if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyset[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; size = setTypeSize(set); /* If count is zero, serve it ASAP to avoid special cases later. */ if (count == 0) { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); return; } @@ -813,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys, } addReply(c,shared.czero); } else { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); } return; } diff --git a/src/t_zset.c b/src/t_zset.c index 2680e76a9..ea6f4b848 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2426,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) { return; } - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) return; /* Sanitize indexes. */ @@ -2438,7 +2438,7 @@ void zrangeGenericCommand(client *c, int reverse) { /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } if (end >= llen) end = llen-1; @@ -2574,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { } /* Ok, lookup the key and get the range */ - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) return; if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { @@ -2594,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (eptr == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } @@ -2661,7 +2661,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (ln == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } @@ -2919,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) { } /* Ok, lookup the key and get the range */ - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) { zslFreeLexRange(&range); @@ -2942,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (eptr == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); zslFreeLexRange(&range); return; } @@ -3006,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (ln == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); zslFreeLexRange(&range); return; } @@ -3160,7 +3160,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey /* No candidate for zpopping, return empty. */ if (!zobj) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } From 9658ff71bf3064780c71646a24925576ef153433 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 13:01:07 +0200 Subject: [PATCH 047/117] Rio: fix flag name, function is never used btw. Thanks to @tnclong for reporting the problem. --- src/rio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rio.h b/src/rio.h index 6199bb039..eb7a05748 100644 --- a/src/rio.h +++ b/src/rio.h @@ -150,7 +150,7 @@ static inline int rioGetReadError(rio *r) { /* Like rioGetReadError() but for write errors. */ static inline int rioGetWriteError(rio *r) { - return (r->flags & RIO_FLAG_READ_ERROR) != 0; + return (r->flags & RIO_FLAG_WRITE_ERROR) != 0; } static inline void rioClearErrors(rio *r) { From 40c8f23fe768f0c369087e9d4567cf8f791da525 Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Wed, 4 Sep 2019 12:14:25 +0000 Subject: [PATCH 048/117] The aarch64 architecture is support normal memory unaligned accesses, so add the UNALIGNED_LE_CPU to the aarch64 . --- src/siphash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/siphash.c b/src/siphash.c index 6b9419031..357741132 100644 --- a/src/siphash.c +++ b/src/siphash.c @@ -58,7 +58,8 @@ int siptlw(int c) { /* Test of the CPU is Little Endian and supports not aligned accesses. * Two interesting conditions to speedup the function that happen to be * in most of x86 servers. */ -#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) +#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) \ + || defined (__aarch64__) || defined (__arm64__) #define UNALIGNED_LE_CPU #endif From 4383958317ed3d0d285dba98950944801e448ba0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 17:20:37 +0200 Subject: [PATCH 049/117] AOF: be future-proof and close the file pointer. Currently useless but we release the fake client, so better to do a full cleanup. Thanks to @TomMD reporting this in #6353. --- src/aof.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/aof.c b/src/aof.c index ae9c4bb68..7237cdfbc 100644 --- a/src/aof.c +++ b/src/aof.c @@ -862,6 +862,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } @@ -892,11 +893,13 @@ uxeof: /* Unexpected AOF end of file. */ } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); exit(1); } From 1f813b3723d39c8f21b6b17b31d14d73beb59486 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 17:55:46 +0200 Subject: [PATCH 050/117] redis-cli: always report server errors on read errors. Before this commit we may have not consumer buffers when a read error is encountered. Such buffers may contain errors that are important clues for the user: for instance a protocol error in the payload we send in pipe mode will cause the server to abort the connection. If the user does not get the protocol error, debugging what is happening can be a nightmare. This commit fixes issue #3756. --- src/redis-cli.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index e363a2795..db53fc7d8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -6724,6 +6724,7 @@ static void pipeMode(void) { /* Handle the readable state: we can read replies from the server. */ if (mask & AE_READABLE) { ssize_t nread; + int read_error = 0; /* Read from socket and feed the hiredis reader. */ do { @@ -6731,7 +6732,8 @@ static void pipeMode(void) { if (nread == -1 && errno != EAGAIN && errno != EINTR) { fprintf(stderr, "Error reading from the server: %s\n", strerror(errno)); - exit(1); + read_error = 1; + break; } if (nread > 0) { redisReaderFeed(reader,ibuf,nread); @@ -6764,6 +6766,11 @@ static void pipeMode(void) { freeReplyObject(reply); } } while(reply); + + /* Abort on read errors. We abort here because it is important + * to consume replies even after a read error: this way we can + * show a potential problem to the user. */ + if (read_error) exit(1); } /* Handle the writable state: we can send protocol to the server. */ From 3984b4fdd9ea122f7b8fbe4a56fdaeb973ccab5b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Sep 2019 13:05:57 +0200 Subject: [PATCH 051/117] Fix handleClientsBlockedOnKeys() names in comments. --- src/blocked.c | 2 +- src/t_list.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 1db657869..9c3907fd4 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -592,7 +592,7 @@ void unblockClientWaitingData(client *c) { * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * - * The list will be finally processed by handleClientsBlockedOnLists() */ + * The list will be finally processed by handleClientsBlockedOnKeys() */ void signalKeyAsReady(redisDb *db, robj *key) { readyList *rl; diff --git a/src/t_list.c b/src/t_list.c index 601ea8899..9bbd61de3 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -606,7 +606,7 @@ void rpoplpushCommand(client *c) { * Blocking POP operations *----------------------------------------------------------------------------*/ -/* This is a helper function for handleClientsBlockedOnLists(). It's work +/* This is a helper function for handleClientsBlockedOnKeys(). It's work * is to serve a specific client (receiver) that is blocked on 'key' * in the context of the specified 'db', doing the following: * From 68e1809589d17eebe766a77fd9e5e8c90afff9f3 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Sep 2019 14:11:37 +0300 Subject: [PATCH 052/117] Fix to module aux data rdb format for backwards compatibility with old check-rdb When implementing the code that saves and loads these aux fields we used rdb format that was added for that in redis 5.0, but then we added the 'when' field which meant that the old redis-check-rdb won't be able to skip these. this fix adds an opcode as if that 'when' is part of the module data. --- src/rdb.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 4e00fad67..e4dfc46b7 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1114,7 +1114,12 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) { if (retval == -1) return -1; io.bytes += retval; - /* write the 'when' so that we can provide it on loading */ + /* write the 'when' so that we can provide it on loading. add a UINT opcode + * for backwards compatibility, everything after the MT needs to be prefixed + * by an opcode. */ + retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_UINT); + if (retval == -1) return -1; + io.bytes += retval; retval = rdbSaveLen(rdb,when); if (retval == -1) return -1; io.bytes += retval; @@ -2132,8 +2137,11 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { * Such data can be potentially be stored both before and after the * RDB keys-values section. */ uint64_t moduleid = rdbLoadLen(rdb,NULL); + int when_opcode = rdbLoadLen(rdb,NULL); int when = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) goto eoferr; + if (when_opcode != RDB_MODULE_OPCODE_UINT) + rdbReportReadError("bad when_opcode"); moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; moduleTypeNameByID(name,moduleid); From f701f7c007665cebf4aa9350a1dcc42da6498434 Mon Sep 17 00:00:00 2001 From: Doug Nelson Date: Thu, 5 Sep 2019 16:25:06 +0100 Subject: [PATCH 053/117] Typo fixes in API documentation --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c529..85674ba0d 100644 --- a/src/module.c +++ b/src/module.c @@ -2389,7 +2389,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * REDISMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a RedisModuleString pointer to pointer, the function just - * reports if the field esists or not and expects an integer pointer + * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of REDISMODULE_HASH_CFIELD: @@ -3288,7 +3288,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { * RedisModule_Realloc() or RedisModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. - * The returned string is not automatically NULL termianted, it is loaded + * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inisde the RDB file. */ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { return moduleLoadString(io,1,lenptr); From 62e74777458bcfbf51f081a2158926be3cc18fca Mon Sep 17 00:00:00 2001 From: suntiawnen Date: Fri, 6 Sep 2019 12:01:44 +0800 Subject: [PATCH 054/117] fix rdb function rdbLoadIntegerObject comment --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index e4dfc46b7..d11fb93f7 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -260,7 +260,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) { /* Loads an integer-encoded object with the specified encoding type "enctype". * The returned value changes according to the flags, see - * rdbGenerincLoadStringObject() for more info. */ + * rdbGenericLoadStringObject() for more info. */ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { int plain = flags & RDB_LOAD_PLAIN; int sds = flags & RDB_LOAD_SDS; From c98af3a5507e75a70111bcf74e3529e3ed2ec3a6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Sep 2019 12:24:26 +0200 Subject: [PATCH 055/117] handleClientsBlockedOnKeys() refactoring. --- src/blocked.c | 403 ++++++++++++++++++++++++++------------------------ 1 file changed, 208 insertions(+), 195 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 9c3907fd4..867f03de6 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -229,6 +229,207 @@ void disconnectAllBlockedClients(void) { } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a list key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnListKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_LIST) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + robj *dstkey = receiver->bpop.target; + int where = (receiver->lastcmd && + receiver->lastcmd->proc == blpopCommand) ? + LIST_HEAD : LIST_TAIL; + robj *value = listTypePop(o,where); + + if (value) { + /* Protect receiver->bpop.target, that will be + * freed by the next unblockClient() + * call. */ + if (dstkey) incrRefCount(dstkey); + unblockClient(receiver); + + if (serveClientBlockedOnList(receiver, + rl->key,dstkey,rl->db,value, + where) == C_ERR) + { + /* If we failed serving the client we need + * to also undo the POP operation. */ + listTypePush(o,value,where); + } + + if (dstkey) decrRefCount(dstkey); + decrRefCount(value); + } else { + break; + } + } + } + + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); + } + /* We don't call signalModifiedKey() as it was already called + * when an element was pushed on the list. */ +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a sorted set key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + unsigned long zcard = zsetLength(o); + + while(numclients-- && zcard) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_ZSET) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + int where = (receiver->lastcmd && + receiver->lastcmd->proc == bzpopminCommand) + ? ZSET_MIN : ZSET_MAX; + unblockClient(receiver); + genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + zcard--; + + /* Replicate the command. */ + robj *argv[2]; + struct redisCommand *cmd = where == ZSET_MIN ? + server.zpopminCommand : + server.zpopmaxCommand; + argv[0] = createStringObject(cmd->name,strlen(cmd->name)); + argv[1] = rl->key; + incrRefCount(rl->key); + propagate(cmd,receiver->db->id, + argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a stream key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + stream *s = o->ptr; + + /* We need to provide the new data arrived on the stream + * to all the clients that are waiting for an offset smaller + * than the current top item. */ + if (de) { + list *clients = dictGetVal(de); + listNode *ln; + listIter li; + listRewind(clients,&li); + + while((ln = listNext(&li))) { + client *receiver = listNodeValue(ln); + if (receiver->btype != BLOCKED_STREAM) continue; + streamID *gt = dictFetchValue(receiver->bpop.keys, + rl->key); + + /* If we blocked in the context of a consumer + * group, we need to resolve the group and update the + * last ID the client is blocked for: this is needed + * because serving other clients in the same consumer + * group will alter the "last ID" of the consumer + * group, and clients blocked in a consumer group are + * always blocked for the ">" ID: we need to deliver + * only new messages and avoid unblocking the client + * otherwise. */ + streamCG *group = NULL; + if (receiver->bpop.xread_group) { + group = streamLookupCG(s, + receiver->bpop.xread_group->ptr); + /* If the group was not found, send an error + * to the consumer. */ + if (!group) { + addReplyError(receiver, + "-NOGROUP the consumer group this client " + "was blocked on no longer exists"); + unblockClient(receiver); + continue; + } else { + *gt = group->last_id; + } + } + + if (streamCompareID(&s->last_id, gt) > 0) { + streamID start = *gt; + start.seq++; /* Can't overflow, it's an uint64_t */ + + /* Lookup the consumer for the group, if any. */ + streamConsumer *consumer = NULL; + int noack = 0; + + if (group) { + consumer = streamLookupConsumer(group, + receiver->bpop.xread_consumer->ptr, + 1); + noack = receiver->bpop.xread_group_noack; + } + + /* Emit the two elements sub-array consisting of + * the name of the stream and the data we + * extracted from it. Wrapped in a single-item + * array, since we have just one key. */ + if (receiver->resp == 2) { + addReplyArrayLen(receiver,1); + addReplyArrayLen(receiver,2); + } else { + addReplyMapLen(receiver,1); + } + addReplyBulk(receiver,rl->key); + + streamPropInfo pi = { + rl->key, + receiver->bpop.xread_group + }; + streamReplyWithRange(receiver,s,&start,NULL, + receiver->bpop.xread_count, + 0, group, consumer, noack, &pi); + + /* Note that after we unblock the client, 'gt' + * and other receiver->bpop stuff are no longer + * valid, so we must do the setup above before + * this call. */ + unblockClient(receiver); + } + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -271,202 +472,14 @@ void handleClientsBlockedOnKeys(void) { /* Serve clients blocked on list key. */ robj *o = lookupKeyWrite(rl->db,rl->key); - if (o != NULL && o->type == OBJ_LIST) { - dictEntry *de; - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - - while(numclients--) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_LIST) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); - - if (value) { - /* Protect receiver->bpop.target, that will be - * freed by the next unblockClient() - * call. */ - if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); - - if (serveClientBlockedOnList(receiver, - rl->key,dstkey,rl->db,value, - where) == C_ERR) - { - /* If we failed serving the client we need - * to also undo the POP operation. */ - listTypePush(o,value,where); - } - - if (dstkey) decrRefCount(dstkey); - decrRefCount(value); - } else { - break; - } - } - } - - if (listTypeLength(o) == 0) { - dbDelete(rl->db,rl->key); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); - } - /* We don't call signalModifiedKey() as it was already called - * when an element was pushed on the list. */ - } - - /* Serve clients blocked on sorted set key. */ - else if (o != NULL && o->type == OBJ_ZSET) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - unsigned long zcard = zsetLength(o); - - while(numclients-- && zcard) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_ZSET) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - int where = (receiver->lastcmd && - receiver->lastcmd->proc == bzpopminCommand) - ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); - genericZpopCommand(receiver,&rl->key,1,where,1,NULL); - zcard--; - - /* Replicate the command. */ - robj *argv[2]; - struct redisCommand *cmd = where == ZSET_MIN ? - server.zpopminCommand : - server.zpopmaxCommand; - argv[0] = createStringObject(cmd->name,strlen(cmd->name)); - argv[1] = rl->key; - incrRefCount(rl->key); - propagate(cmd,receiver->db->id, - argv,2,PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(argv[0]); - decrRefCount(argv[1]); - } - } - } - - /* Serve clients blocked on stream key. */ - else if (o != NULL && o->type == OBJ_STREAM) { - dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = o->ptr; - - /* We need to provide the new data arrived on the stream - * to all the clients that are waiting for an offset smaller - * than the current top item. */ - if (de) { - list *clients = dictGetVal(de); - listNode *ln; - listIter li; - listRewind(clients,&li); - - while((ln = listNext(&li))) { - client *receiver = listNodeValue(ln); - if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = dictFetchValue(receiver->bpop.keys, - rl->key); - - /* If we blocked in the context of a consumer - * group, we need to resolve the group and update the - * last ID the client is blocked for: this is needed - * because serving other clients in the same consumer - * group will alter the "last ID" of the consumer - * group, and clients blocked in a consumer group are - * always blocked for the ">" ID: we need to deliver - * only new messages and avoid unblocking the client - * otherwise. */ - streamCG *group = NULL; - if (receiver->bpop.xread_group) { - group = streamLookupCG(s, - receiver->bpop.xread_group->ptr); - /* If the group was not found, send an error - * to the consumer. */ - if (!group) { - addReplyError(receiver, - "-NOGROUP the consumer group this client " - "was blocked on no longer exists"); - unblockClient(receiver); - continue; - } else { - *gt = group->last_id; - } - } - - if (streamCompareID(&s->last_id, gt) > 0) { - streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ - - /* Lookup the consumer for the group, if any. */ - streamConsumer *consumer = NULL; - int noack = 0; - - if (group) { - consumer = streamLookupConsumer(group, - receiver->bpop.xread_consumer->ptr, - 1); - noack = receiver->bpop.xread_group_noack; - } - - /* Emit the two elements sub-array consisting of - * the name of the stream and the data we - * extracted from it. Wrapped in a single-item - * array, since we have just one key. */ - if (receiver->resp == 2) { - addReplyArrayLen(receiver,1); - addReplyArrayLen(receiver,2); - } else { - addReplyMapLen(receiver,1); - } - addReplyBulk(receiver,rl->key); - - streamPropInfo pi = { - rl->key, - receiver->bpop.xread_group - }; - streamReplyWithRange(receiver,s,&start,NULL, - receiver->bpop.xread_count, - 0, group, consumer, noack, &pi); - - /* Note that after we unblock the client, 'gt' - * and other receiver->bpop stuff are no longer - * valid, so we must do the setup above before - * this call. */ - unblockClient(receiver); - } - } - } + if (o != NULL) { + if (o->type == OBJ_LIST) + serveClientsBlockedOnListKey(o,rl); + else if (o->type == OBJ_ZSET) + serveClientsBlockedOnSortedSetKey(o,rl); + else if (o->type == OBJ_STREAM) + serveClientsBlockedOnStreamKey(o,rl); } /* Free this item. */ From e645c794cf654da8d13cf46e838b12f6ab9f5f94 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Sep 2019 19:42:10 +0200 Subject: [PATCH 056/117] ACL: protect MULTI/EXEC transactions after rules change. --- src/multi.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/multi.c b/src/multi.c index 71090d8ed..f885fa19c 100644 --- a/src/multi.c +++ b/src/multi.c @@ -175,7 +175,19 @@ void execCommand(client *c) { must_propagate = 1; } - call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + int acl_retval = ACLCheckCommandPerm(c); + if (acl_retval != ACL_OK) { + addReplyErrorFormat(c, + "-NOPERM ACLs rules changed between the moment the " + "transaction was accumulated and the EXEC call. " + "This command is no longer allowed for the " + "following reason: %s", + (acl_retval == ACL_DENIED_CMD) ? + "no permission to execute the command or subcommand" : + "no permission to touch the specified keys"); + } else { + call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + } /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; From ecf8dc6ff6fed0a73e1588416db3fcfa9516ee79 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:21:34 +0200 Subject: [PATCH 057/117] ACL: add slightly modified version of sha256.c for password hashing. memory.h include removed, types substituted with stdint types. --- src/Makefile | 2 +- src/sha256.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/sha256.h | 35 ++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/sha256.c create mode 100644 src/sha256.h diff --git a/src/Makefile b/src/Makefile index b6cc69e2f..198d85cd5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -164,7 +164,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/sha256.c b/src/sha256.c new file mode 100644 index 000000000..d644d2d4e --- /dev/null +++ b/src/sha256.c @@ -0,0 +1,158 @@ +/********************************************************************* +* Filename: sha256.c +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Implementation of the SHA-256 hashing algorithm. + SHA-256 is one of the three algorithms in the SHA2 + specification. The others, SHA-384 and SHA-512, are not + offered in this implementation. + Algorithm specification can be found here: + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf + This implementation uses little endian byte order. +*********************************************************************/ + +/*************************** HEADER FILES ***************************/ +#include +#include +#include "sha256.h" + +/****************************** MACROS ******************************/ +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +/**************************** VARIABLES *****************************/ +static const WORD k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +/*********************** FUNCTION DEFINITIONS ***********************/ +void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) +{ + WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + e = ctx->state[4]; + f = ctx->state[5]; + g = ctx->state[6]; + h = ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; + t2 = EP0(a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX *ctx) +{ + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) +{ + WORD i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX *ctx, BYTE hash[]) +{ + WORD i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = ctx->bitlen; + ctx->data[62] = ctx->bitlen >> 8; + ctx->data[61] = ctx->bitlen >> 16; + ctx->data[60] = ctx->bitlen >> 24; + ctx->data[59] = ctx->bitlen >> 32; + ctx->data[58] = ctx->bitlen >> 40; + ctx->data[57] = ctx->bitlen >> 48; + ctx->data[56] = ctx->bitlen >> 56; + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} diff --git a/src/sha256.h b/src/sha256.h new file mode 100644 index 000000000..dc53ead2b --- /dev/null +++ b/src/sha256.h @@ -0,0 +1,35 @@ +/********************************************************************* +* Filename: sha256.h +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Defines the API for the corresponding SHA1 implementation. +*********************************************************************/ + +#ifndef SHA256_H +#define SHA256_H + +/*************************** HEADER FILES ***************************/ +#include +#include + +/****************************** MACROS ******************************/ +#define SHA256_BLOCK_SIZE 32 // SHA256 outputs a 32 byte digest + +/**************************** DATA TYPES ****************************/ +typedef uint8_t BYTE; // 8-bit byte +typedef uint32_t WORD; // 32-bit word + +typedef struct { + BYTE data[64]; + WORD datalen; + unsigned long long bitlen; + WORD state[8]; +} SHA256_CTX; + +/*********************** FUNCTION DECLARATIONS **********************/ +void sha256_init(SHA256_CTX *ctx); +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len); +void sha256_final(SHA256_CTX *ctx, BYTE hash[]); + +#endif // SHA256_H From 036c83d8ec94639491b5d29b6e072e618eead4e3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:33:22 +0200 Subject: [PATCH 058/117] ACL: SHA256 based password hashing function implemented. --- src/acl.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/acl.c b/src/acl.c index a2ee65dd0..aea4fe97f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -28,6 +28,7 @@ */ #include "server.h" +#include "sha256.h" #include /* ============================================================================= @@ -139,6 +140,25 @@ int time_independent_strcmp(char *a, char *b) { return diff; /* If zero strings are the same. */ } +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(sds cleartext) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[SHA256_BLOCK_SIZE*2]; + char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,SHA256_BLOCK_SIZE*2); +} + /* ============================================================================= * Low level ACL API * ==========================================================================*/ From fa2ed64ba15fcd034c49c9c7e34006bfaed3998b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:54:57 +0200 Subject: [PATCH 059/117] ACL: store hashed passwords in memory. Note that this breaks API compatibility with Redis < 6: CONFIG GET requirepass Will no longer return a cleartext password as well, but the SHA256 hash of the password set. --- src/acl.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/acl.c b/src/acl.c index aea4fe97f..2cd729e77 100644 --- a/src/acl.c +++ b/src/acl.c @@ -142,14 +142,14 @@ int time_independent_strcmp(char *a, char *b) { /* Given an SDS string, returns the SHA256 hex representation as a * new SDS string. */ -sds ACLHashPassword(sds cleartext) { +sds ACLHashPassword(unsigned char *cleartext, size_t len) { SHA256_CTX ctx; unsigned char hash[SHA256_BLOCK_SIZE]; char hex[SHA256_BLOCK_SIZE*2]; char *cset = "0123456789abcdef"; sha256_init(&ctx); - sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_update(&ctx,(unsigned char*)cleartext,len); sha256_final(&ctx,hash); for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { @@ -721,13 +721,16 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); } else if (op[0] == '>') { - sds newpass = sdsnewlen(op+1,oplen-1); + sds newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ - if (ln == NULL) listAddNodeTail(u->passwords,newpass); + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; } else if (op[0] == '<') { - sds delpass = sdsnewlen(op+1,oplen-1); + sds delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { @@ -744,7 +747,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); /* Avoid re-adding the same pattern multiple times. */ - if (ln == NULL) listAddNodeTail(u->patterns,newpat); + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { @@ -899,11 +905,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) { listIter li; listNode *ln; listRewind(u->passwords,&li); + sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr)); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - if (!time_independent_strcmp(password->ptr, thispass)) + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); return C_OK; + } } + sdsfree(hashed); /* If we reached this point, no password matched. */ errno = EINVAL; From 33e35c3562dcf58fce8cafeb6afb45f00fbd2e03 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:01:39 +0200 Subject: [PATCH 060/117] RESP3: implement lua.setresp(). --- src/scripting.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 032bfdf10..d5da14332 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -42,7 +42,7 @@ char *redisProtocolToLuaType_Int(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -132,9 +132,9 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '$': p = redisProtocolToLuaType_Bulk(lua,reply); break; case '+': p = redisProtocolToLuaType_Status(lua,reply); break; case '-': p = redisProtocolToLuaType_Error(lua,reply); break; - case '*': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '%': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '~': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; + case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; } return p; } @@ -182,7 +182,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply) { return p+2; } -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *p = strchr(reply+1,'\r'); long long mbulklen; int j = 0; @@ -859,6 +859,25 @@ int luaLogCommand(lua_State *lua) { return 0; } +/* redis.setresp() */ +int luaSetResp(lua_State *lua) { + int argc = lua_gettop(lua); + + if (argc != 1) { + lua_pushstring(lua, "redis.setresp() requires one argument."); + return lua_error(lua); + } + + int resp = lua_tonumber(lua,-argc); + if (resp != 2 && resp != 3) { + lua_pushstring(lua, "RESP version must be 2 or 3."); + return lua_error(lua); + } + + server.lua_client->resp = resp; + return 0; +} + /* --------------------------------------------------------------------------- * Lua engine initialization and reset. * ------------------------------------------------------------------------- */ @@ -986,6 +1005,11 @@ void scriptingInit(int setup) { lua_pushcfunction(lua,luaLogCommand); lua_settable(lua,-3); + /* redis.setresp */ + lua_pushstring(lua,"setresp"); + lua_pushcfunction(lua,luaSetResp); + lua_settable(lua,-3); + lua_pushstring(lua,"LOG_DEBUG"); lua_pushnumber(lua,LL_DEBUG); lua_settable(lua,-3); @@ -1379,8 +1403,9 @@ void evalGenericCommand(client *c, int evalsha) { luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); - /* Select the right DB in the context of the Lua client */ + /* Set the Lua client database and protocol. */ selectDb(server.lua_client,c->db->id); + server.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ /* Set a hook in order to be able to stop the script execution if it * is running for too much time. From 1eecd5f2de75d69ddeab714501206d45b712262c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:19:10 +0200 Subject: [PATCH 061/117] RESP3: Lua debugger support for printing sets and maps. --- src/scripting.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index d5da14332..68c7ce006 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2078,6 +2078,8 @@ char *ldbRedisProtocolToHuman_Int(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2092,6 +2094,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '+': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '-': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; + case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; + case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; } return p; } @@ -2146,6 +2150,40 @@ char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"~(",2); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,")",1); + return p; +} + +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"{",1); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + *o = sdscatlen(*o," => ",4); + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,"}",1); + return p; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From e563362f1ddcbbe0c7efdd6d772f80d8b2fc5ddb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:38:35 +0200 Subject: [PATCH 062/117] RESP3: Lua parsing should depend on lua client, not lua caller. We want all the scripts to run in RESP2 mode by default. It's up to the caller to switch to V3 using redis.setresp() if it is really needed. This way most scripts written for past Redis versions will continue to work with Redis >= 6 even if the client is in RESP3 mode. --- src/scripting.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 68c7ce006..ee37ac23f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -188,7 +188,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { int j = 0; string2ll(reply+1,p-reply-1,&mbulklen); - if (server.lua_caller->resp == 2 || atype == '*') { + if (server.lua_client->resp == 2 || atype == '*') { p += 2; if (mbulklen == -1) { lua_pushboolean(lua,0); @@ -200,7 +200,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { p = redisProtocolToLuaType(lua,p); lua_settable(lua,-3); } - } else if (server.lua_caller->resp == 3) { + } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays * follow the above RESP2 code path. */ p += 2; From 6184d97323c225b1fb213cbd2f2ff5e0b6dd89ca Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 15 Sep 2019 21:16:30 +0100 Subject: [PATCH 063/117] [add] improved performance of RM_ReplyWithSimpleString and RM_ReplyWithError by making usage addReplyProto instead of addReplySds --- src/module.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c529..ad7e6864c 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,19 +1120,6 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } -/* Reply with an error or simple string (status message). Used to implement - * ReplyWithSimpleString() and ReplyWithError(). - * The function always returns REDISMODULE_OK. */ -int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - sds strmsg = sdsnewlen(prefix,1); - strmsg = sdscat(strmsg,msg); - strmsg = sdscatlen(strmsg,"\r\n",2); - addReplySds(c,strmsg); - return REDISMODULE_OK; -} - /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1148,7 +1135,13 @@ int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - return replyWithStatus(ctx,err,"-"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(err); + addReplyProto(c,"-",1); + addReplyProto(c,err,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1157,7 +1150,13 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - return replyWithStatus(ctx,msg,"+"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"+",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with an array type of 'len' elements. However 'len' other calls From dde8e3f8f45020f7c004bfdd6dfb72124bdc5d06 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 11:49:42 +0200 Subject: [PATCH 064/117] RESP3: report set/map as nested tables to Lua. --- src/scripting.c | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index ee37ac23f..39f75abf0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -202,9 +202,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays - * follow the above RESP2 code path. */ + * follow the above RESP2 code path. Note that those are represented + * as a table with the "map" or "set" field populated with the actual + * table representing the set or the map type. */ p += 2; lua_newtable(lua); + lua_pushstring(lua,atype == '%' ? "map" : "set"); + lua_newtable(lua); for (j = 0; j < mbulklen; j++) { p = redisProtocolToLuaType(lua,p); if (atype == '%') { @@ -214,6 +218,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } lua_settable(lua,-3); } + lua_settable(lua,-3); } return p; } @@ -310,6 +315,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { * Error are returned as a single element table with 'err' field. * Status replies are returned as single element table with 'ok' * field. */ + + /* Handle error reply. */ lua_pushstring(lua,"err"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -321,8 +328,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { lua_pop(lua,2); return; } + lua_pop(lua,1); /* Discard field name pushed before. */ - lua_pop(lua,1); + /* Handle status reply. */ lua_pushstring(lua,"ok"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -332,24 +340,25 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); lua_pop(lua,1); - } else { - void *replylen = addReplyDeferredLen(c); - int j = 1, mbulklen = 0; - - lua_pop(lua,1); /* Discard the 'ok' field value we popped */ - while(1) { - lua_pushnumber(lua,j++); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TNIL) { - lua_pop(lua,1); - break; - } - luaReplyToRedisReply(c, lua); - mbulklen++; - } - setDeferredArrayLen(c,replylen,mbulklen); + return; } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle the array reply. */ + void *replylen = addReplyDeferredLen(c); + int j = 1, mbulklen = 0; + while(1) { + lua_pushnumber(lua,j++); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNIL) { + lua_pop(lua,1); + break; + } + luaReplyToRedisReply(c, lua); + mbulklen++; + } + setDeferredArrayLen(c,replylen,mbulklen); break; default: addReplyNull(c); From 347f85bddb9050d27a747e4b32de3956cea4f6af Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:15:39 +0200 Subject: [PATCH 065/117] RESP3: handle map Lua -> Redis conversion. --- src/scripting.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 39f75abf0..cd55000d4 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -297,6 +297,8 @@ void luaSortArray(lua_State *lua) { * Lua reply to Redis reply conversion functions. * ------------------------------------------------------------------------- */ +/* Reply to client 'c' converting the top element in the Lua stack to a + * Redis reply. As a side effect the element is consumed from the stack. */ void luaReplyToRedisReply(client *c, lua_State *lua) { int t = lua_type(lua,-1); @@ -339,7 +341,29 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { sdsmapchars(ok,"\r\n"," ",2); addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); - lua_pop(lua,1); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle map reply. */ + lua_pushstring(lua,"map"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int maplen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, value */ + luaReplyToRedisReply(c, lua); /* Return value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + maplen++; + } + setDeferredMapLen(c,replylen,maplen); + lua_pop(lua,2); return; } lua_pop(lua,1); /* Discard field name pushed before. */ From 5303bf6890b8ce5479adc5a9c0cb7c045f1f6980 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:19:19 +0200 Subject: [PATCH 066/117] RESP3: handle set Lua -> Redis conversion. --- src/scripting.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index cd55000d4..2c90bb7ae 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -368,6 +368,28 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle set reply. */ + lua_pushstring(lua,"set"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int setlen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, true */ + lua_pop(lua,1); /* Discard the boolean value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + setlen++; + } + setDeferredSetLen(c,replylen,setlen); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle the array reply. */ void *replylen = addReplyDeferredLen(c); int j = 1, mbulklen = 0; From 547f3a29ab41fa16d2c207d5cef2e7dd20469f67 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 17:49:40 +0200 Subject: [PATCH 067/117] RESP3: implement new NULL representation parsing in Lua. --- src/scripting.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 2c90bb7ae..564dad8ca 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -43,6 +43,7 @@ char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -135,6 +136,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '_': p = redisProtocolToLuaType_Null(lua,reply); break; } return p; } @@ -223,6 +225,12 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { return p; } +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,0); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From 65a2bc7807c70428cf2c6834f64ea608a7f75721 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:18:17 +0200 Subject: [PATCH 068/117] RESP3: change behavior of Lua returning true/false for RESP3. Here we introduce a change in the way we convert values from Lua to Redis when RESP3 is selected: this is possible without breaking the fact we can return directly what a command returned, because there is no Redis command in RESP2 that returns true or false to Lua, so the conversion in the case of RESP2 is totally arbitrary. When a script is written selecting RESP3 from Lua, it totally makes sense to change such behavior and return RESP3 true/false when Lua true/false is returned. --- src/scripting.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 564dad8ca..3ad228a67 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -315,7 +315,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); break; case LUA_TBOOLEAN: - addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); + if (server.lua_client->resp == 2) + addReply(c,lua_toboolean(lua,-1) ? shared.cone : + shared.null[c->resp]); + else + addReplyBool(c,lua_toboolean(lua,-1)); break; case LUA_TNUMBER: addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); From ff27682d78103392c4a22be68a24756dca90527d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:36:16 +0200 Subject: [PATCH 069/117] RESP3: convert RESP3 null as Lua nil. Implement RESP3->Lua bools. --- src/scripting.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 3ad228a67..9ab1e7ab3 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -44,6 +44,7 @@ char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -137,6 +138,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); } return p; } @@ -227,7 +229,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { char *p = strchr(reply+1,'\r'); - lua_pushboolean(lua,0); + lua_pushnil(lua); + return p+2; +} + +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,tf == 't'); return p+2; } From 1fdefd825d2e08090a2dbda9c9d23f11ea679fe4 Mon Sep 17 00:00:00 2001 From: Okada Haruki Date: Tue, 17 Sep 2019 06:18:01 +0900 Subject: [PATCH 070/117] Fix typo --- src/hyperloglog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 5d4afaa25..a44d15646 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) { if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ if (isHLLObjectOrReply(c,o) != C_OK) return; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(registers,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) { hdr = o->ptr; if (hdr->encoding == HLL_DENSE) use_dense = 1; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(max,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); From 607ec36df04526f6a95aaaac701ed17387c7f204 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 18:57:24 +0200 Subject: [PATCH 071/117] RESP3: bool and null values in RESP -> human readable conversion. --- src/scripting.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 9ab1e7ab3..b76ff278a 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2155,6 +2155,8 @@ char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2171,6 +2173,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; + case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; + case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; } return p; } @@ -2259,6 +2263,21 @@ char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(null)",6); + return p+2; +} + +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + if (reply[1] == 't') + *o = sdscatlen(*o,"#true",5); + else + *o = sdscatlen(*o,"#false",6); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From f3ac14411563128589448e17371efe7d8c0d8786 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:08:33 +0200 Subject: [PATCH 072/117] RESP3: double -> human readable conversion. --- src/scripting.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index b76ff278a..456210533 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2157,6 +2157,7 @@ char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2175,6 +2176,7 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; + case ',': p = ldbRedisProtocolToHuman_Double(o,reply); break; } return p; } @@ -2278,6 +2280,13 @@ char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { return p+2; } +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(double) ",9); + *o = sdscatlen(*o,reply+1,p-reply-1); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From 11678504f710021c3a6bea7c0568972d74e388f8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:20:30 +0200 Subject: [PATCH 073/117] RESP3: RESP3 double -> Lua conversion. --- src/scripting.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 456210533..7230f7438 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -45,6 +45,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -139,6 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; } @@ -239,6 +241,27 @@ char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { return p+2; } +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + char buf[MAX_LONG_DOUBLE_CHARS+1]; + size_t len = p-reply-1; + double d; + + if (len <= MAX_LONG_DOUBLE_CHARS) { + memcpy(buf,reply+1,len); + buf[len] = '\0'; + d = strtod(buf,NULL); /* We expect a valid representation. */ + } else { + d = 0; + } + + lua_newtable(lua); + lua_pushstring(lua,"double"); + lua_pushnumber(lua,d); + lua_settable(lua,-3); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From eb464f737cb21f47d4a4ae89d8d7d84b46e4dc62 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:26:46 +0200 Subject: [PATCH 074/117] RESP3: Lua double -> RESP3 conversion. --- src/scripting.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 7230f7438..2153233f5 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -389,6 +389,17 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle double reply. */ + lua_pushstring(lua,"double"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNUMBER) { + addReplyDouble(c,lua_tonumber(lua,-1)); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle map reply. */ lua_pushstring(lua,"map"); lua_gettable(lua,-2); From 238fd6546cbe58d031fb4121f495db37932a041a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:28:39 +0200 Subject: [PATCH 075/117] RESP3: Verbatim conversion test in the LOLWUT command. redis-cli is currently not able to handle it after going in RESP3 mode, because of hiredis limitations. --- src/lolwut.c | 3 ++- src/lolwut5.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index 19cbcf642..ba7e1069e 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -43,7 +43,8 @@ void lolwutUnstableCommand(client *c) { sds rendered = sdsnew("Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); } void lolwutCommand(client *c) { diff --git a/src/lolwut5.c b/src/lolwut5.c index 8408b378d..52a98c0d7 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -277,6 +277,7 @@ void lolwut5Command(client *c) { "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); lwFreeCanvas(canvas); } From bbc1faa9d633db46a8064b2deb33f54d2097899b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:33:01 +0200 Subject: [PATCH 076/117] RESP3: Use verbatim in INFO output. --- src/server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index fb308e5f8..7882b0d99 100644 --- a/src/server.c +++ b/src/server.c @@ -4391,7 +4391,9 @@ void infoCommand(client *c) { addReply(c,shared.syntaxerr); return; } - addReplyBulkSds(c, genRedisInfoString(section)); + sds info = genRedisInfoString(section); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } void monitorCommand(client *c) { From 90f06b04b009d8a47218a7f08b41cc9318103fb8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:46:11 +0200 Subject: [PATCH 077/117] RESP3: Use verbatim in DEBUG HTSTATS / HTSTATS-KEY. --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..93173f3b9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -638,7 +638,8 @@ NULL dictGetStats(buf,sizeof(buf),server.db[dbid].expires); stats = sdscat(stats,buf); - addReplyBulkSds(c,stats); + addReplyVerbatim(c,stats,sdslen(stats),"txt"); + sdsfree(stats); } else if (!strcasecmp(c->argv[1]->ptr,"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -665,7 +666,7 @@ NULL } else { char buf[4096]; dictGetStats(buf,sizeof(buf),ht); - addReplyBulkCString(c,buf); + addReplyVerbatim(c,buf,strlen(buf),"txt"); } } else if (!strcasecmp(c->argv[1]->ptr,"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); From b98016ca055b55c841d63dc175f3ded2368389da Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:48:14 +0200 Subject: [PATCH 078/117] RESP3: Use verbatim in MEMORY subcommands. --- src/object.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index fb011d311..4534d7c06 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,13 +1440,15 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyBulkSds(c, info); + addReplyVerbatim(c,info,sdslen(info),"txt") + sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); #endif } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { sds report = getMemoryDoctorReport(); - addReplyBulkSds(c,report); + addReplyVerbatim(c,report,sdslen(report),"txt"); + sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { #if defined(USE_JEMALLOC) char tmp[32]; From 2c96a98ed8d552c3ec9f1f63f589747737d4f77c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:51:15 +0200 Subject: [PATCH 079/117] RESP3: Use verbatim in CLUSTER subcommands. --- src/cluster.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index a2615fdc0..1e7dcd50e 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4252,7 +4252,9 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - addReplyBulkSds(c,clusterGenNodesDescription(0)); + sds nodes = clusterGenNodesDescription(0); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4494,10 +4496,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); From 90cf427a5ca6f92c52f09ba9bc9dddd66a00316f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:52:13 +0200 Subject: [PATCH 080/117] RESP3: Use verbatim in CLIENT LIST. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7555ca77d..a959d557a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1990,7 +1990,7 @@ NULL return; } sds o = getAllClientsInfoString(type); - addReplyBulkCBuffer(c,o,sdslen(o)); + addReplyVerbatim(c,o,sdslen(o),"txt"); sdsfree(o); } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) { /* CLIENT REPLY ON|OFF|SKIP */ From 747b4dd62f6061c735bef16b87225575e1c911ec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:53:22 +0200 Subject: [PATCH 081/117] RESP3: Use verbatim in LATENCY subcommands. --- src/latency.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/latency.c b/src/latency.c index 33aa1245b..b834da5c7 100644 --- a/src/latency.c +++ b/src/latency.c @@ -599,7 +599,7 @@ NULL event = dictGetKey(de); graph = latencyCommandGenSparkeline(event,ts); - addReplyBulkCString(c,graph); + addReplyVerbatim(c,graph,sdslen(graph),"txt"); sdsfree(graph); } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { /* LATENCY LATEST */ @@ -608,7 +608,7 @@ NULL /* LATENCY DOCTOR */ sds report = createLatencyReport(); - addReplyBulkCBuffer(c,report,sdslen(report)); + addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { /* LATENCY RESET */ From dd077faf49d66019741805010b65918fa7fde24e Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:14:36 +0800 Subject: [PATCH 082/117] Fix bad handling of unexpected option while loading config "lua-replicate-commands". --- src/config.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config.c b/src/config.c index a72df2e78..ae4d4a0e6 100644 --- a/src/config.c +++ b/src/config.c @@ -672,6 +672,9 @@ void loadServerConfigFromString(char *config) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { server.lua_always_replicate_commands = yesnotoi(argv[1]); + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { From c6454647838a0fd56ed807de9ddf45bdeb5cfd90 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:37:23 +0800 Subject: [PATCH 083/117] RESP3: Fix function redisProtocolToLuaType about RESP3->Lua bools. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 2153233f5..fa896de21 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -139,7 +139,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; - case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; From 4429a40317c386e756f5f539f79c56502207f1d3 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 20 Sep 2019 01:11:20 -0700 Subject: [PATCH 084/117] Fix compilation error --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index 4534d7c06..697429b84 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,7 +1440,7 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyVerbatim(c,info,sdslen(info),"txt") + addReplyVerbatim(c,info,sdslen(info),"txt"); sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); From 5cb591ca40ca424779dd97a47dece41831fce97a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:18:59 +0200 Subject: [PATCH 085/117] Add useless break for uniformity / future protection. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index fa896de21..9ac8af2e7 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -140,7 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; - case ',': p = redisProtocolToLuaType_Double(lua,reply); + case ',': p = redisProtocolToLuaType_Double(lua,reply); break; } return p; } From e7444fde2d1776f408a6f006c21ee97f5afcd9cd Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:44:32 +0200 Subject: [PATCH 086/117] Remove redundant statement in config.c. Thanks to @guybe7 for spotting the error in the original PR I merged. --- src/config.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index ae4d4a0e6..fe002bac2 100644 --- a/src/config.c +++ b/src/config.c @@ -671,8 +671,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) + == -1) + { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && From 31a20f0fbb6cdd59fa404bd22dcd62806e88f50e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:46:35 +0200 Subject: [PATCH 087/117] Make config.c always_replicate_commands more uniform. Better if it resembles the other similar code paths. --- src/config.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fe002bac2..d37e3c566 100644 --- a/src/config.c +++ b/src/config.c @@ -671,10 +671,10 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) - == -1) - { - err = "argument must be 'yes' or 'no'"; goto loaderr; + server.lua_always_replicate_commands = yesnotoi(argv[1]); + if (server.lua_always_replicate_commands == -1) { + err = "argument must be 'yes' or 'no'"; + goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) From a84c87cc95a347b4b3a2caa9c7ca1aa0415e21c0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 20:06:47 +0200 Subject: [PATCH 088/117] hiredis updated to master version. --- deps/hiredis/.gitignore | 1 + deps/hiredis/.travis.yml | 74 ++- deps/hiredis/CHANGELOG.md | 15 +- deps/hiredis/CMakeLists.txt | 90 ++++ deps/hiredis/Makefile | 106 ++++- deps/hiredis/README.md | 3 +- deps/hiredis/adapters/libevent.h | 112 ++++- deps/hiredis/appveyor.yml | 7 +- deps/hiredis/async.c | 176 +++++--- deps/hiredis/async.h | 8 + deps/hiredis/async_private.h | 72 +++ deps/hiredis/examples/CMakeLists.txt | 46 ++ deps/hiredis/examples/example-libevent-ssl.c | 73 +++ deps/hiredis/examples/example-libevent.c | 15 +- deps/hiredis/examples/example-ssl.c | 97 ++++ deps/hiredis/examples/example.c | 17 +- deps/hiredis/hiredis.c | 253 ++++++----- deps/hiredis/hiredis.h | 103 ++++- deps/hiredis/hiredis.pc.in | 11 + deps/hiredis/hiredis_ssl.h | 53 +++ deps/hiredis/hiredis_ssl.pc.in | 12 + deps/hiredis/net.c | 122 +++-- deps/hiredis/net.h | 4 + deps/hiredis/read.c | 28 +- deps/hiredis/read.h | 5 +- deps/hiredis/sds.c | 2 +- deps/hiredis/sds.h | 31 +- deps/hiredis/sockcompat.c | 248 ++++++++++ deps/hiredis/sockcompat.h | 91 ++++ deps/hiredis/ssl.c | 448 +++++++++++++++++++ deps/hiredis/test.c | 93 +++- deps/hiredis/test.sh | 70 +++ deps/hiredis/win32.h | 18 +- 33 files changed, 2178 insertions(+), 326 deletions(-) create mode 100644 deps/hiredis/CMakeLists.txt create mode 100644 deps/hiredis/async_private.h create mode 100644 deps/hiredis/examples/CMakeLists.txt create mode 100644 deps/hiredis/examples/example-libevent-ssl.c create mode 100644 deps/hiredis/examples/example-ssl.c create mode 100644 deps/hiredis/hiredis.pc.in create mode 100644 deps/hiredis/hiredis_ssl.h create mode 100644 deps/hiredis/hiredis_ssl.pc.in create mode 100644 deps/hiredis/sockcompat.c create mode 100644 deps/hiredis/sockcompat.h create mode 100644 deps/hiredis/ssl.c create mode 100755 deps/hiredis/test.sh diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index c44b5c537..8e50b5434 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index faf2ce684..dd8e0e73d 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -26,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" + +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index a7fe3ac11..d1d37e515 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -12,6 +12,16 @@ compare to other values, casting might be necessary or can be removed, if casting was applied before. +### 0.x.x (unreleased) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + ### 0.14.0 (2018-09-25) * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) @@ -50,8 +60,9 @@ * Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 * Fix warnings, when compiled with -Wshadow * Make hiredis compile in Cygwin on Windows, now CI-tested - -**BREAKING CHANGES**: +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt new file mode 100644 index 000000000..9e78894f3 --- /dev/null +++ b/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 06ca99468..25ac15464 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -39,7 +45,7 @@ export REDIS_TEST_CONFIG CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) REAL_LDFLAGS=$(LDFLAGS) @@ -49,12 +55,30 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) @@ -66,40 +90,61 @@ ifeq ($(uname_S),Darwin) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -116,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -133,32 +178,33 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c INSTALL?= cp -pPR @@ -175,6 +221,20 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 01223ea59..c0b432f07 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 7d2bef180..a4952776c 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_free(e->rev); - event_free(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml index 819efbd58..5b43fdbeb 100644 --- a/deps/hiredis/appveyor.yml +++ b/deps/hiredis/appveyor.yml @@ -5,8 +5,9 @@ environment: CC: gcc - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 clone_depth: 1 @@ -20,4 +21,4 @@ install: build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -344,9 +330,15 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -358,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -408,7 +403,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, assert(reply->element[2]->type == REDIS_REPLY_INTEGER); /* Unset subscribed flag only when no pipelined pending subscribe. */ - if (reply->element[2]->integer == 0 + if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; @@ -524,6 +519,18 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { } } +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + /* This function should be called when the socket is readable. * It processes all replies that can be read and executes their callbacks. */ @@ -539,28 +546,13 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } + c->funcs->async_read(ac); } -void redisAsyncHandleWrite(redisAsyncContext *ac) { +void redisAsyncWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); int done = 0; - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { @@ -575,6 +567,51 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { } } +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + c->funcs->async_write(ac); +} + +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); +} + /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ static const char *nextArgument(const char *start, const char **str, size_t *len) { @@ -714,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 740555c24..4f6b3b783 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -57,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -81,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -106,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -113,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h new file mode 100644 index 000000000..d0133ae18 --- /dev/null +++ b/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 000000000..dd3a313ac --- /dev/null +++ b/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 000000000..1021113b9 --- /dev/null +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index d333c22b7..1fe71ae4e 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c new file mode 100644 index 000000000..81f4648c6 --- /dev/null +++ b/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 4d494c55a..0e93fc8b3 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 0947d1ed7..282595bdb 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,10 +41,20 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); @@ -112,19 +121,31 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(r->vtype,buf,3); + r->vtype[3] = '\0'; + memcpy(buf+4,str,len-4); + buf[len-4] = '\0'; + } else { + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(buf,str,len); + buf[len] = '\0'; + } r->str = buf; r->len = len; @@ -138,7 +159,7 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; r = createReplyObject(task->type); @@ -649,29 +670,30 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); + redisNetClose(c); sdsfree(c->obuf); redisReaderFree(c->reader); @@ -680,12 +702,16 @@ void redisFree(redisContext *c) { free(c->unix_sock.path); free(c->timeout); free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -694,10 +720,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -718,112 +747,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -853,22 +877,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); + return REDIS_ERR; } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -883,21 +900,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 47d7982e9..69dc39c5e 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -35,7 +35,11 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ @@ -74,6 +78,12 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -92,6 +102,8 @@ typedef struct redisReply { size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING and REDIS_REPLY_DOUBLE (in additionl to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -111,14 +123,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -139,8 +230,12 @@ typedef struct redisContext { /* For non-blocking connect */ struct sockadr *saddr; size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -151,7 +246,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -167,7 +262,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in new file mode 100644 index 000000000..140b040f1 --- /dev/null +++ b/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h new file mode 100644 index 000000000..f844f9548 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 000000000..588a978a5 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index a4b3abc6d..e5f40b0a4 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -34,36 +34,64 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; @@ -79,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -101,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -108,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -119,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); + return REDIS_ERR; + } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,12 +249,12 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -230,7 +267,7 @@ static int redisContextWaitReady(redisContext *c, long msec) { } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -277,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -291,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -360,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -401,16 +446,14 @@ addrretry: } /* For repeat connection */ - if (c->saddr) { - free(c->saddr); - } + free(c->saddr); c->saddr = malloc(p->ai_addrlen); memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; } else if (errno == EINPROGRESS) { if (blocking) { @@ -424,7 +467,7 @@ addrretry: if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { @@ -471,8 +514,9 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; if (redisCreateSocket(c,AF_UNIX) < 0) @@ -499,9 +543,11 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_UNIX; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -516,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a11594e68..a4393c06b 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -37,6 +37,10 @@ #include "hiredis.h" +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); + int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index cc0f3cc72..b9853ea9a 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -31,10 +31,10 @@ #include "fmacros.h" #include -#include #include #ifndef _MSC_VER #include +#include #endif #include #include @@ -44,6 +44,7 @@ #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -294,9 +295,9 @@ static int processLineItem(redisReader *r) { buf[len] = '\0'; if (strcasecmp(buf,",inf") == 0) { - d = 1.0/0.0; /* Positive infinite. */ + d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -1.0/0.0; /* Nevative infinite. */ + d = -INFINITY; /* Nevative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -379,10 +380,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -430,7 +439,7 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || elements > INT_MAX) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -523,6 +532,9 @@ static int processItem(redisReader *r) { case '#': cur->type = REDIS_REPLY_BOOL; break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -543,6 +555,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: @@ -657,8 +670,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index f3d075843..58105312a 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -55,12 +56,12 @@ #define REDIS_REPLY_ERROR 6 #define REDIS_REPLY_DOUBLE 7 #define REDIS_REPLY_BOOL 8 -#define REDIS_REPLY_VERB 9 #define REDIS_REPLY_MAP 9 #define REDIS_REPLY_SET 10 #define REDIS_REPLY_ATTR 11 #define REDIS_REPLY_PUSH 12 #define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 #define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ @@ -79,7 +80,7 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 44777b10c..6cf75841c 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1035,7 +1035,7 @@ sds *sdssplitargs(const char *line, int *argc) { s_free(vector); return NULL; } - + vector = new_vector; vector[*argc] = current; (*argc)++; diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 13be75a9f..3f9a96457 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -132,20 +135,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -156,21 +159,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -200,16 +203,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c new file mode 100644 index 000000000..4cc2f414f --- /dev/null +++ b/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h new file mode 100644 index 000000000..56006c163 --- /dev/null +++ b/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c new file mode 100644 index 000000000..78ab9e43e --- /dev/null +++ b/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 79cff4308..8668e1856 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -13,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -33,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -93,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -121,9 +149,21 @@ static redisContext *do_connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -360,7 +400,8 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); - test("Set error when array > INT_MAX: "); +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); ret = redisReaderGetReply(reader,&reply); @@ -369,7 +410,6 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); -#if LLONG_MAX > SIZE_MAX test("Set error when bulk > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); @@ -434,22 +474,23 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; struct addrinfo hints = {.ai_family = AF_INET}; struct addrinfo *ai_tmp = NULL; - const char *bad_domain = "idontexist.com"; - int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp); + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); if (rv != 0) { // Address does *not* exist test("Returns error when host cannot be resolved: "); // First see if this domain name *actually* resolves to NXDOMAIN - c = redisConnect("dontexist.com", 6379); + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); test_cond( c->err == REDIS_ERR_OTHER && (strcmp(c->errstr, "Name or service not known") == 0 || - strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || @@ -574,7 +615,8 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -583,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -591,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -894,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -922,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh new file mode 100755 index 000000000..2cab9e6fb --- /dev/null +++ b/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ From bd32b8b2a8bc743187bf6fc98de6163660268afe Mon Sep 17 00:00:00 2001 From: valentino Date: Sat, 21 Sep 2019 20:58:57 +0300 Subject: [PATCH 089/117] DISCARD should not fail during OOM discard command should not fail during OOM, otherwise client MULTI state will not be cleared. --- src/server.c | 2 +- tests/unit/multi.tcl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..3737c3352 100644 --- a/src/server.c +++ b/src/server.c @@ -3411,7 +3411,7 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand))) { + (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 6655bf62c..9fcef71d6 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -306,4 +306,18 @@ start_server {tags {"multi"}} { } close_replication_stream $repl } + + test {DISCARD should not fail during OOM} { + set rd [redis_deferring_client] + $rd config set maxmemory 1 + assert {[$rd read] eq {OK}} + r multi + catch {r set x 1} e + assert_match {OOM*} $e + r discard + $rd config set maxmemory 0 + assert {[$rd read] eq {OK}} + $rd close + r ping + } {PONG} } From 8ceffc04558648c7a3758f46dab51debe2bd3d0a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:39:42 +0200 Subject: [PATCH 090/117] RESP3: implementation of verbatim output with TTY target. --- src/redis-cli.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index db53fc7d8..d08dfeb5f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -819,10 +819,17 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) { out = sdscatprintf(out,"(double) %s\n",r->str); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: /* If you are producing output for the standard output we want - * a more interesting output with quoted characters and so forth */ - out = sdscatrepr(out,r->str,r->len); - out = sdscat(out,"\n"); + * a more interesting output with quoted characters and so forth, + * unless it's a verbatim string type. */ + if (r->type == REDIS_REPLY_STRING) { + out = sdscatrepr(out,r->str,r->len); + out = sdscat(out,"\n"); + } else { + out = sdscatlen(out,r->str,r->len); + out = sdscat(out,"\n"); + } break; case REDIS_REPLY_NIL: out = sdscat(out,"(nil)\n"); From d097eebf0694987d48d59fbbd2da7eeff15c0a03 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:41:04 +0200 Subject: [PATCH 091/117] hiredis udpated (RESP3 WIP). --- deps/hiredis/hiredis.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 282595bdb..abd94c01d 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -128,15 +128,16 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { - buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ if (buf == NULL) { freeReplyObject(r); return NULL; } - memcpy(r->vtype,buf,3); + memcpy(r->vtype,str,3); r->vtype[3] = '\0'; - memcpy(buf+4,str,len-4); + memcpy(buf,str+4,len-4); buf[len-4] = '\0'; + r->len = len-4; } else { buf = malloc(len+1); if (buf == NULL) { @@ -145,9 +146,9 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len } memcpy(buf,str,len); buf[len] = '\0'; + r->len = len; } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; From cd446eb161b02db9a9affea8ba916c1c1ad6fa76 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:42:57 +0200 Subject: [PATCH 092/117] RESP3: varbatim handling for other redis-cli outputs. --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index d08dfeb5f..9a5f81723 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -968,6 +968,7 @@ static sds cliFormatReplyRaw(redisReply *r) { break; case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: if (r->type == REDIS_REPLY_STATUS && config.eval_ldb) { /* The Lua debugger replies with arrays of simple (status) * strings. We colorize the output for more fun if this @@ -1021,6 +1022,7 @@ static sds cliFormatReplyCSV(redisReply *r) { out = sdscatprintf(out,"%lld",r->integer); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: From 24a78adf1063653adc80187139e01233fdb3d5b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:47:36 +0200 Subject: [PATCH 093/117] redis-cli: AUTH can now have 3 arguments as well. --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 9a5f81723..38a0b0b7f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1222,7 +1222,8 @@ static int cliSendCommand(int argc, char **argv, long repeat) { if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) { config.dbnum = atoi(argv[1]); cliRefreshPrompt(); - } else if (!strcasecmp(command,"auth") && argc == 2) { + } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) + { cliSelect(); } } From ddb735f03a6b43700a8c870254fcf4e00a6a7da6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:55:05 +0200 Subject: [PATCH 094/117] redis-cli: ability to start a session in RESP3 mode. --- src/redis-cli.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 38a0b0b7f..47bd11a8a 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -230,6 +230,7 @@ static struct config { int verbose; clusterManagerCommand cluster_manager_command; int no_auth_warning; + int resp3; } config; /* User preferences. */ @@ -751,6 +752,21 @@ static int cliSelect(void) { return REDIS_ERR; } +/* Select RESP3 mode if redis-cli was started with the -3 option. */ +static int cliSwitchProto(void) { + redisReply *reply; + if (config.resp3 == 0) return REDIS_OK; + + reply = redisCommand(context,"HELLO 3"); + if (reply != NULL) { + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR; + freeReplyObject(reply); + return result; + } + return REDIS_ERR; +} + /* Connect to the server. It is possible to pass certain flags to the function: * CC_FORCE: The connection is performed even if there is already * a connected socket. @@ -788,11 +804,13 @@ static int cliConnect(int flags) { * errors. */ anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); - /* Do AUTH and select the right DB. */ + /* Do AUTH, select the right DB, switch to RESP3 if needed. */ if (cliAuth() != REDIS_OK) return REDIS_ERR; if (cliSelect() != REDIS_OK) return REDIS_ERR; + if (cliSwitchProto() != REDIS_OK) + return REDIS_ERR; } return REDIS_OK; } @@ -1449,6 +1467,8 @@ static int parseOptions(int argc, char **argv) { printf("redis-cli %s\n", version); sdsfree(version); exit(0); + } else if (!strcmp(argv[i],"-3")) { + config.resp3 = 1; } else if (CLUSTER_MANAGER_MODE() && argv[i][0] != '-') { if (config.cluster_manager_command.argc == 0) { int j = i + 1; @@ -1529,6 +1549,7 @@ static void usage(void) { " -i When -r is used, waits seconds per command.\n" " It is possible to specify sub-second times like -i 0.1.\n" " -n Database number.\n" +" -3 Start session in RESP3 protocol mode.\n" " -x Read last argument from STDIN.\n" " -d Multi-bulk delimiter in for raw formatting (default: \\n).\n" " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" @@ -1543,7 +1564,9 @@ static void usage(void) { " --csv is specified, or if you redirect the output to a non\n" " TTY, it samples the latency for 1 second (you can use\n" " -i to change the interval), then produces a single output\n" -" and exits.\n" +" and exits.\n",version); + + fprintf(stderr, " --latency-history Like --latency but tracking latency changes over time.\n" " Default time interval is 15 sec. Change it using -i.\n" " --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n" @@ -1578,7 +1601,7 @@ static void usage(void) { " --help Output this help and exit.\n" " --version Output version and exit.\n" "\n", - version, REDIS_CLI_DEFAULT_PIPE_TIMEOUT); + REDIS_CLI_DEFAULT_PIPE_TIMEOUT); /* Using another fprintf call to avoid -Woverlength-strings compile warning */ fprintf(stderr, "Cluster Manager Commands:\n" From c7481c6af1441f272fa4f8b6b9b559ae5aa64b73 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:36:06 +0200 Subject: [PATCH 095/117] redis-cli: CSV and RAW target for more RESP3 types. --- src/redis-cli.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 47bd11a8a..7374526e5 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1006,9 +1006,15 @@ static sds cliFormatReplyRaw(redisReply *r) { out = sdscatlen(out,r->str,r->len); } break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "(true)" : "(false)"); + break; case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_ARRAY: for (i = 0; i < r->elements; i++) { if (i > 0) out = sdscat(out,config.mb_delim); @@ -1017,6 +1023,19 @@ static sds cliFormatReplyRaw(redisReply *r) { sdsfree(tmp); } break; + case REDIS_REPLY_MAP: + for (i = 0; i < r->elements; i += 2) { + if (i > 0) out = sdscat(out,config.mb_delim); + tmp = cliFormatReplyRaw(r->element[i]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + + out = sdscatlen(out," ",1); + tmp = cliFormatReplyRaw(r->element[i+1]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + } + break; default: fprintf(stderr,"Unknown reply type: %d\n", r->type); exit(1); @@ -1039,14 +1058,21 @@ static sds cliFormatReplyCSV(redisReply *r) { case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_STRING: case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: - out = sdscat(out,"NIL"); + out = sdscat(out,"NULL"); + break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "true" : "false"); break; case REDIS_REPLY_ARRAY: + case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */ for (i = 0; i < r->elements; i++) { sds tmp = cliFormatReplyCSV(r->element[i]); out = sdscatlen(out,tmp,sdslen(tmp)); From fddbae80faf422562ce4d5db305de4a688090490 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:57:13 +0200 Subject: [PATCH 096/117] redis-cli: support for ACL style user/pass AUTH. --- src/redis-cli.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7374526e5..c183155cb 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -218,6 +218,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; char prompt[128]; @@ -729,8 +730,13 @@ static int cliAuth(void) { redisReply *reply; if (config.auth == NULL) return REDIS_OK; - reply = redisCommand(context,"AUTH %s",config.auth); + if (config.user == NULL) + reply = redisCommand(context,"AUTH %s",config.auth); + else + reply = redisCommand(context,"AUTH %s %s",config.user,config.auth); if (reply != NULL) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr,"Warning: AUTH failed\n"); freeReplyObject(reply); return REDIS_OK; } @@ -1350,8 +1356,12 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; - } else if (!strcmp(argv[i],"-a") && !lastarg) { + } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) + && !lastarg) + { config.auth = argv[++i]; + } else if (!strcmp(argv[i],"--user") && !lastarg) { + config.user = argv[++i]; } else if (!strcmp(argv[i],"-u") && !lastarg) { parseRedisUri(argv[++i]); } else if (!strcmp(argv[i],"--raw")) { @@ -1570,6 +1580,8 @@ static void usage(void) { " You can also use the " REDIS_CLI_AUTH_ENV " environment\n" " variable to pass this password more safely\n" " (if both are used, this argument takes predecence).\n" +" -user Used to send ACL style 'AUTH username pass'. Needs -a.\n" +" -pass Alias of -a for consistency with the new --user option.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -2409,7 +2421,12 @@ static int clusterManagerNodeConnect(clusterManagerNode *node) { * errors. */ anetKeepAlive(NULL, node->context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); if (config.auth) { - redisReply *reply = redisCommand(node->context,"AUTH %s",config.auth); + redisReply *reply; + if (config.user == NULL) + reply = redisCommand(node->context,"AUTH %s", config.auth); + else + reply = redisCommand(node->context,"AUTH %s %s", + config.user,config.auth); int ok = clusterManagerCheckRedisReply(node, reply, NULL); if (reply != NULL) freeReplyObject(reply); if (!ok) return 0; @@ -7737,6 +7754,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.user = NULL; config.eval = NULL; config.eval_ldb = 0; config.eval_ldb_end = 0; From 26452e0d85c9b2e2c6a523b16109ba0087c1321b Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 23 Sep 2019 23:45:31 +0100 Subject: [PATCH 097/117] [fix] un-refactor the code. [perf] replyWithStatus now makes usage of addReplyProto --- src/module.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/module.c b/src/module.c index ad7e6864c..067a69928 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,6 +1120,19 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } +/* Reply with an error or simple string (status message). Used to implement + * ReplyWithSimpleString() and ReplyWithError(). + * The function always returns REDISMODULE_OK. */ +int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"-",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; +} + /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1135,13 +1148,7 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(err); - addReplyProto(c,"-",1); - addReplyProto(c,err,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,err,"-"); } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1150,13 +1157,7 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"+",1); - addReplyProto(c,msg,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,msg,"+"); } /* Reply with an array type of 'len' elements. However 'len' other calls From c9a39e9bc5bc40757b50408bda2a95c319829357 Mon Sep 17 00:00:00 2001 From: "Mike A. Owens" Date: Mon, 23 Sep 2019 19:24:09 -0400 Subject: [PATCH 098/117] Seed SipHash with 128-bit key SipHash expects a 128-bit key, and we were indeed generating 128-bits, but restricting them to hex characters 0-9a-f, effectively giving us only 4 bits-per-byte of key material, and 64 bits overall. Now, we skip the hex conversion and supply 128 bits of unfiltered random data. --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..fc9809b1c 100644 --- a/src/server.c +++ b/src/server.c @@ -4787,9 +4787,9 @@ int main(int argc, char **argv) { srand(time(NULL)^getpid()); gettimeofday(&tv,NULL); - char hashseed[16]; - getRandomHexChars(hashseed,sizeof(hashseed)); - dictSetHashFunctionSeed((uint8_t*)hashseed); + uint8_t hashseed[16]; + getRandomBytes(hashseed,sizeof(hashseed)); + dictSetHashFunctionSeed(hashseed); server.sentinel_mode = checkForSentinelMode(argc,argv); initServerConfig(); ACLInit(); /* The ACL subsystem must be initialized ASAP because the From 53e3b994f07cfbcd4eee3dec76c7b180c75c709f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 17:45:05 +0200 Subject: [PATCH 099/117] ACL: fix ##6408, default user state affecting all the connections. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..2bdfd1640 100644 --- a/src/server.c +++ b/src/server.c @@ -3341,9 +3341,10 @@ int processCommand(client *c) { /* Check if the user is authenticated. This check is skipped in case * the default user is flagged as "nopass" and is active. */ - int auth_required = !(DefaultUser->flags & USER_FLAG_NOPASS) && + int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || + DefaultUser->flags & USER_FLAG_DISABLED) && !c->authenticated; - if (auth_required || DefaultUser->flags & USER_FLAG_DISABLED) { + if (auth_required) { /* AUTH and HELLO are valid even in non authenticated state. */ if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { flagTransaction(c); From 73eac241659e7dfdf81f41b7242ec1c83a0b966f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 18:08:11 +0200 Subject: [PATCH 100/117] Modify #6401 changes to fit 80 cols. --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 42dc8f1e4..51106abe0 100644 --- a/src/server.c +++ b/src/server.c @@ -3412,7 +3412,10 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { + (c->flags & CLIENT_MULTI && + c->cmd->proc != execCommand && + c->cmd->proc != discardCommand))) + { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; From 02ddcc0c78e7537cf4cabb4fc593771f901b01db Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 25 Sep 2019 17:28:42 +0100 Subject: [PATCH 101/117] [fix] fixed the un-refactor bug. --- src/module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 067a69928..d02e2708c 100644 --- a/src/module.c +++ b/src/module.c @@ -1126,9 +1126,10 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"-",1); - addReplyProto(c,msg,len); + const size_t msgLen = strlen(msg); + const size_t prefixLen = strlen(prefix); + addReplyProto(c,prefix,prefixLen); + addReplyProto(c,msg,msgLen); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From cf2dc0779133c8be10c73d22345521c3e9204a75 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 12:18:39 +0200 Subject: [PATCH 102/117] INFO: more info about loaded modules. Related to #6024. --- src/module.c | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 71fdd6b74..b2bddd316 100644 --- a/src/module.c +++ b/src/module.c @@ -52,7 +52,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ - int options; /* Moduile options and capabilities. */ + int options; /* Module options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -5363,9 +5363,36 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper for genModulesInfoString(): given a list of modules, return + * am SDS string in the form "[modulename|modulename2|...]" */ +sds genModulesInfoStringRenderModulesList(list *l) { + listIter li; + listNode *ln; + listRewind(l,&li); + sds output = sdsnew("["); + while((ln = listNext(&li))) { + RedisModule *module = ln->value; + output = sdscat(output,module->name); + } + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + +/* Helper for genModulesInfoString(): render module options as an SDS string. */ +sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { + sds output = sdsnew("["); + if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) + output = sdscat(output,"handle-io-errors|"); + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + + /* Helper function for the INFO command: adds loaded modules as to info's * output. - * + * * After the call, the passed sds info string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ sds genModulesInfoString(sds info) { @@ -5376,7 +5403,17 @@ sds genModulesInfoString(sds info) { sds name = dictGetKey(de); struct RedisModule *module = dictGetVal(de); - info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + sds usedby = genModulesInfoStringRenderModulesList(module->usedby); + sds using = genModulesInfoStringRenderModulesList(module->using); + sds options = genModulesInfoStringRenderModuleOptions(module); + info = sdscatprintf(info, + "module:name=%s,ver=%d,api=%d,filters=%d," + "usedby=%s,using=%s,options=%s\r\n", + name, module->ver, module->apiver, + (int)listLength(module->filters), usedby, using, options); + sdsfree(usedby); + sdsfree(using); + sdsfree(options); } dictReleaseIterator(di); return info; From 2add4524c533ec1fdafccf28300e56d0ea6e16b7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 26 Sep 2019 15:16:34 +0300 Subject: [PATCH 103/117] Fix lastbgsave_status, when new child signal handler get intended kill And add a test for that. --- src/server.c | 9 ++++++++- tests/integration/rdb.tcl | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 31418e01b..c0e59c86c 100644 --- a/src/server.c +++ b/src/server.c @@ -1910,6 +1910,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); + /* sigKillChildHandler catches the signal and calls exit(), but we + * must make sure not to flag lastbgsave_status, etc incorrectly. */ + if (exitcode == SIGUSR1) { + bysignal = SIGUSR1; + exitcode = 1; + } + if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", @@ -4578,7 +4585,7 @@ static void sigKillChildHandler(int sig) { UNUSED(sig); /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(1); + exitFromChild(SIGUSR1); } void setupChildSignalHandlers(void) { diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 58a098edc..b364291ee 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -115,3 +115,17 @@ start_server_and_kill_it [list "dir" $server_path] { } } } + +start_server {} { + test {Test FLUSHALL aborts bgsave} { + r config set rdb-key-save-delay 1000 + r debug populate 1000 + r bgsave + assert_equal [s rdb_bgsave_in_progress] 1 + r flushall + after 200 + assert_equal [s rdb_bgsave_in_progress] 0 + # make sure the server is still writable + r set x xx + } +} \ No newline at end of file From 5d78cf769c005151abe15f77dbf8dd7a4393aa6b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 16:14:21 +0200 Subject: [PATCH 104/117] BGREWRITEAOF: improve the generic error message. --- src/aof.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 7237cdfbc..91d0d1a76 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1630,7 +1630,8 @@ void bgrewriteaofCommand(client *c) { } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } From c9cf8a3c5f790d4297a209af4d6c1ea1b04b2139 Mon Sep 17 00:00:00 2001 From: nikhilajayk Date: Thu, 26 Sep 2019 21:51:49 +0530 Subject: [PATCH 105/117] Added cluster host and protected mode variables --- utils/create-cluster/README | 2 +- utils/create-cluster/create-cluster | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/create-cluster/README b/utils/create-cluster/README index e682f6dc9..37a3080db 100644 --- a/utils/create-cluster/README +++ b/utils/create-cluster/README @@ -16,7 +16,7 @@ To create a cluster, follow these steps: number of instances you want to create. 2. Use "./create-cluster start" in order to run the instances. 3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that -an actual Redis cluster will be created. +an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP) 4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. In order to stop a cluster: diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 468f924a4..9ffd462ae 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -1,10 +1,12 @@ #!/bin/bash # Settings +CLUSTER_HOST=127.0.0.1 PORT=30000 TIMEOUT=2000 NODES=6 REPLICAS=1 +PROTECTED_MODE=yes # You may want to put the above config parameters into config.sh in order to # override the defaults without modifying this script. @@ -22,7 +24,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes + ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes done exit 0 fi @@ -32,7 +34,7 @@ then HOSTS="" while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) - HOSTS="$HOSTS 127.0.0.1:$PORT" + HOSTS="$HOSTS $CLUSTER_HOST:$PORT" done ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS exit 0 From e885d7401882f4773fc3bad53eaba02e9a558dcd Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:39:40 +0200 Subject: [PATCH 106/117] Modules fork: improve SIGUSR1 handling, fix include. We can't expect SIGUSR1 to have any specific value range, so let's define an exit code that we can handle in a special way. This also fixes an #include that is not standard. --- src/module.c | 2 +- src/server.c | 14 ++++++++++---- src/server.h | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 854989e73..d3b37a3d8 100644 --- a/src/module.c +++ b/src/module.c @@ -31,7 +31,7 @@ #include "cluster.h" #include "rdb.h" #include -#include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" diff --git a/src/server.c b/src/server.c index f38ed7897..046694e1f 100644 --- a/src/server.c +++ b/src/server.c @@ -1913,8 +1913,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); /* sigKillChildHandler catches the signal and calls exit(), but we - * must make sure not to flag lastbgsave_status, etc incorrectly. */ - if (exitcode == SIGUSR1) { + * must make sure not to flag lastbgsave_status, etc incorrectly. + * We could directly terminate the child process via SIGUSR1 + * without handling it, but in this case Valgrind will log an + * annoying error. */ + if (exitcode == SERVER_CHILD_NOERROR_RETVAL) { bysignal = SIGUSR1; exitcode = 1; } @@ -4618,11 +4621,14 @@ void setupSignalHandlers(void) { return; } +/* This is the signal handler for children process. It is currently useful + * in order to track the SIGUSR1, that we send to a child in order to terminate + * it in a clean way, without the parent detecting an error and stop + * accepting writes because of a write error condition. */ static void sigKillChildHandler(int sig) { UNUSED(sig); - /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(SIGUSR1); + exitFromChild(SERVER_CHILD_NOERROR_RETVAL); } void setupChildSignalHandlers(void) { diff --git a/src/server.h b/src/server.h index d132cf09c..c701d6d21 100644 --- a/src/server.h +++ b/src/server.h @@ -179,6 +179,14 @@ typedef long long mstime_t; /* millisecond time type. */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 +/* Children process will exit with this status code to signal that the + * process terminated without an error: this is useful in order to kill + * a saving child (RDB or AOF one), without triggering in the parent the + * write protection that is normally turned on on write errors. + * Usually children that are terminated with SIGUSR1 will exit with this + * special code. */ +#define SERVER_CHILD_NOERROR_RETVAL 255 + /* Instantaneous metrics tracking. */ #define STATS_METRIC_SAMPLES 16 /* Number of samples per metric. */ #define STATS_METRIC_COMMAND 0 /* Number of commands executed. */ From afc8866d661708364fa8f57dd65a2baa5e169a6b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:37 +0200 Subject: [PATCH 107/117] Improve error message in BGSAVE. --- src/rdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index d9164b21c..e1cedf6f9 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2590,9 +2590,9 @@ void bgsaveCommand(client *c) { addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "Another BG operation is in progress: can't BGSAVE right now. " - "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " - "possible."); + "Another child process is active (AOF?): can't BGSAVE right now. " + "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " + "possible."); } } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) { addReplyStatus(c,"Background saving started"); From eba38bc3a9317ada4dd7add999e60e8499dfff82 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:58 +0200 Subject: [PATCH 108/117] TerminateModuleForkChild(): use wait4 for safety. In theory currently there is only one active child, but the API may change or for bugs in the implementation we may have several (it was like that for years because of a bug). Better to wait for a specific pid and avoid consuing other pending children information. --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d3b37a3d8..cee7a85a9 100644 --- a/src/module.c +++ b/src/module.c @@ -5193,7 +5193,8 @@ void TerminateModuleForkChild(int wait) { serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { - while(wait3(&statloc,0,NULL) != server.module_child_pid); + while(wait4(server.module_child_pid,&statloc,0,NULL) != + server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ server.module_child_pid = -1; From 1ef0c2f6307297046b800b5f1c8a32b7efc5657a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:03:09 +0200 Subject: [PATCH 109/117] Function renamed hasForkChild() -> hasActiveChildProcess(). --- src/aof.c | 10 +++++----- src/db.c | 2 +- src/defrag.c | 2 +- src/module.c | 2 +- src/rdb.c | 6 +++--- src/replication.c | 4 ++-- src/server.c | 14 +++++++------- src/server.h | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/aof.c b/src/aof.c index 8bc6c543d..4e6af7c1c 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,7 +264,7 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (hasForkChild() && server.aof_child_pid == -1) { + if (hasActiveChildProcess() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { @@ -395,7 +395,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -491,7 +491,7 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && hasForkChild()) + if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess()) return; /* Perform the fsync if needed. */ @@ -1563,7 +1563,7 @@ void aofClosePipes(void) { int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); if ((childpid = redisFork()) == 0) { @@ -1607,7 +1607,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/db.c b/src/db.c index a46e0251f..afedc6aec 100644 --- a/src/db.c +++ b/src/db.c @@ -60,7 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index 93c6a4619..50f6b41f2 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (hasForkChild()) + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index cee7a85a9..4e6ad6cf9 100644 --- a/src/module.c +++ b/src/module.c @@ -5157,7 +5157,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; - if (hasForkChild()) { + if (hasActiveChildProcess()) { return -1; } diff --git a/src/rdb.c b/src/rdb.c index e1cedf6f9..e430bcd58 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1336,7 +1336,7 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); @@ -2417,7 +2417,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { pid_t childpid; int pipefds[2]; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2584,7 +2584,7 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); diff --git a/src/replication.c b/src/replication.c index 8039e06ae..76090a9e5 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,7 +751,7 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, @@ -2930,7 +2930,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/server.c b/src/server.c index 046694e1f..b2ab653a2 100644 --- a/src/server.c +++ b/src/server.c @@ -1449,13 +1449,13 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (!hasForkChild()) + if (!hasActiveChildProcess()) dictEnableResize(); else dictDisableResize(); } -int hasForkChild() { +int hasActiveChildProcess() { return server.rdb_child_pid != -1 || server.aof_child_pid != -1 || server.module_child_pid != -1; @@ -1697,7 +1697,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1894,14 +1894,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (hasForkChild() || ldbPendingChildren()) + if (hasActiveChildProcess() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1975,7 +1975,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - !hasForkChild() && + !hasActiveChildProcess() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2033,7 +2033,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) diff --git a/src/server.h b/src/server.h index c701d6d21..2ab3f2b45 100644 --- a/src/server.h +++ b/src/server.h @@ -1818,7 +1818,7 @@ void receiveChildInfo(void); /* Fork helpers */ int redisFork(); -int hasForkChild(); +int hasActiveChildProcess(); void sendChildCOWInfo(int ptype, char *pname); /* acl.c -- Authentication related prototypes. */ From 7aaedf9192c75ec1d502daa7277dae81dade6e97 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:17:47 +0200 Subject: [PATCH 110/117] TerminateModuleForkChild(): move safety checks there. We don't want that the API could be used directly in an unsafe way, without checking if there is an active child. Now the safety checks are moved directly in the function performing the operations. --- src/module.c | 35 ++++++++++++++++++----------------- src/server.c | 2 +- src/server.h | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 4e6ad6cf9..27d5eb863 100644 --- a/src/module.c +++ b/src/module.c @@ -5031,7 +5031,7 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi ln = listSearchKey(moduleCommandFilters,filter); if (!ln) return REDISMODULE_ERR; listDelNode(moduleCommandFilters,ln); - + ln = listSearchKey(ctx->module->filters,filter); if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); @@ -5154,8 +5154,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) * Return: -1 on failure, on success the parent process will get a positive PID * of the child, and the child process will get 0. */ -int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) -{ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; if (hasActiveChildProcess()) { return -1; @@ -5181,14 +5180,20 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) /* Call from the child process when you want to terminate it. * retcode will be provided to the done handler executed on the parent process. */ -int RM_ExitFromChild(int retcode) -{ +int RM_ExitFromChild(int retcode) { sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); exitFromChild(retcode); return REDISMODULE_OK; } -void TerminateModuleForkChild(int wait) { +/* Kill the active module forked child, if there is one active and the + * pid matches, and returns C_OK. Otherwise if there is no active module + * child or the pid does not match, return C_ERR without doing anything. */ +void TerminateModuleForkChild(int child_pid, int wait) { + /* Module child should be active and pid should match. */ + if (server.module_child_pid == -1 || + server.module_child_pid != child_pid) return C_ERR; + int statloc; serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); @@ -5202,24 +5207,20 @@ void TerminateModuleForkChild(int wait) { moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); + return C_OK; } /* Can be used to kill the forked child process from the parent process. * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; +int RM_KillForkChild(int child_pid) { /* Kill module child, wait for child exit. */ - TerminateModuleForkChild(1); - return REDISMODULE_OK; + if (TerminateModuleForkChild(child_pid,1) == C_OK) + return REDISMODULE_OK; + else + return REDISMODULE_ERR; } -void ModuleForkDoneHandler(int exitcode, int bysignal) -{ +void ModuleForkDoneHandler(int exitcode, int bysignal) { serverLog(LL_NOTICE, "Module fork exited pid: %d, retcode: %d, bysignal: %d", server.module_child_pid, exitcode, bysignal); diff --git a/src/server.c b/src/server.c index b2ab653a2..a2af7824d 100644 --- a/src/server.c +++ b/src/server.c @@ -3589,7 +3589,7 @@ int prepareForShutdown(int flags) { /* Kill module child if there is one. */ if (server.module_child_pid != -1) { serverLog(LL_WARNING,"There is a module fork child. Killing it!"); - TerminateModuleForkChild(0); + TerminateModuleForkChild(server.module_child_pid,0); } if (server.aof_state != AOF_OFF) { diff --git a/src/server.h b/src/server.h index 2ab3f2b45..52abef4b8 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int wait); +void TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From 48aae0d0db1c029100d6c37c99f25bae911d93db Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:23:07 +0200 Subject: [PATCH 111/117] TerminateModuleForkChild(): fix function prototype. --- src/module.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 27d5eb863..217478705 100644 --- a/src/module.c +++ b/src/module.c @@ -5189,7 +5189,7 @@ int RM_ExitFromChild(int retcode) { /* Kill the active module forked child, if there is one active and the * pid matches, and returns C_OK. Otherwise if there is no active module * child or the pid does not match, return C_ERR without doing anything. */ -void TerminateModuleForkChild(int child_pid, int wait) { +int TerminateModuleForkChild(int child_pid, int wait) { /* Module child should be active and pid should match. */ if (server.module_child_pid == -1 || server.module_child_pid != child_pid) return C_ERR; diff --git a/src/server.h b/src/server.h index 52abef4b8..91c7219a0 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int child_pid, int wait); +int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From 795d1029b4c54068ad32b82e53c03393b90672fe Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:32:52 +0200 Subject: [PATCH 112/117] Fix memory leak in RM_UnregisterCommandFilter(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 217478705..98d4db97f 100644 --- a/src/module.c +++ b/src/module.c @@ -5036,6 +5036,8 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); + zfree(filter); + return REDISMODULE_OK; } From bd0ddcdc3bc652be02ca8f4f51a758acf3541d79 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:33:21 +0200 Subject: [PATCH 113/117] Fix memory leak in moduleLoadFromQueue(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 98d4db97f..351ffcc45 100644 --- a/src/module.c +++ b/src/module.c @@ -5334,6 +5334,8 @@ void moduleLoadFromQueue(void) { void moduleFreeModuleStructure(struct RedisModule *module) { listRelease(module->types); listRelease(module->filters); + listRelease(module->usedby); + listRelease(module->using); sdsfree(module->name); zfree(module); } From 9ec77b6a7a9d9ec9283017a0ea354ad6a733faae Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:42:38 +0200 Subject: [PATCH 114/117] moduleRDBLoadError(): io->ctx may be NULL. The correct way to access the module about a given IO context is to deference io->type->module, since io->ctx is only populated if the user requests an explicit context from an IO object. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 351ffcc45..8e9ffbf87 100644 --- a/src/module.c +++ b/src/module.c @@ -3177,7 +3177,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { + if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } From 0660ff3696b51e1b39fcf7416a57291748a6a5ef Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 10:58:15 +0200 Subject: [PATCH 115/117] Change a bit the style of #6385. --- src/module.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 9c2bed08f..13353df2c 100644 --- a/src/module.c +++ b/src/module.c @@ -1149,10 +1149,8 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t msgLen = strlen(msg); - const size_t prefixLen = strlen(prefix); - addReplyProto(c,prefix,prefixLen); - addReplyProto(c,msg,msgLen); + addReplyProto(c,prefix,strlen(prefix)); + addReplyProto(c,msg,strlen(msg)); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From 70dbce427369e8241850a62f63f356eecffdef38 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 17:17:36 +0200 Subject: [PATCH 116/117] Fix comments aesthetics. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index a2af7824d..79c458ee9 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,8 +4058,9 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* this is the total RSS overhead, including fragmentation, */ - mh->total_frag_bytes, /* named so for backwards compatibility */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation. */ + mh->total_frag_bytes, /* Named so for backwards compatibility. */ freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves, From 700cc17a6a5f456932c7bab072319cd82bced2c6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 18:37:59 +0200 Subject: [PATCH 117/117] Clarify a comment about memory total_frag field. --- src/server.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 79c458ee9..cacce6dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,9 +4058,11 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* This is the total RSS overhead, including - fragmentation. */ - mh->total_frag_bytes, /* Named so for backwards compatibility. */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation, but not just it. This field + (and the next one) is named like that just + for backward compatibility. */ + mh->total_frag_bytes, freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves,