From 58735ed74ecc5ac0ad4290e5e09f65320a352584 Mon Sep 17 00:00:00 2001 From: Wander Hillen Date: Fri, 16 Mar 2018 09:59:14 +0100 Subject: [PATCH 001/320] Fix typos, add some periods --- src/replication.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 8c01bfb51..c5c3618a5 100644 --- a/src/replication.c +++ b/src/replication.c @@ -255,7 +255,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { while((ln = listNext(&li))) { client *slave = ln->value; - /* Don't feed slaves that are still waiting for BGSAVE to start */ + /* Don't feed slaves that are still waiting for BGSAVE to start. */ if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue; /* Feed slaves that are waiting for the initial SYNC (so these commands @@ -294,7 +294,7 @@ void replicationFeedSlavesFromMasterStream(list *slaves, char *buf, size_t bufle while((ln = listNext(&li))) { client *slave = ln->value; - /* Don't feed slaves that are still waiting for BGSAVE to start */ + /* Don't feed slaves that are still waiting for BGSAVE to start. */ if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue; addReplyString(slave,buf,buflen); } @@ -584,7 +584,7 @@ int startBgsaveForReplication(int mincapa) { } /* If we failed to BGSAVE, remove the slaves waiting for a full - * resynchorinization from the list of salves, inform them with + * resynchronization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ if (retval == C_ERR) { serverLog(LL_WARNING,"BGSAVE for replication failed"); @@ -604,7 +604,7 @@ int startBgsaveForReplication(int mincapa) { } /* If the target is socket, rdbSaveToSlavesSockets() already setup - * the salves for a full resync. Otherwise for disk target do it now.*/ + * the slaves for a full resync. Otherwise for disk target do it now.*/ if (!socket_target) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { From 48bc22ec4c2597467865388dad536606fd820347 Mon Sep 17 00:00:00 2001 From: Wander Hillen Date: Fri, 16 Mar 2018 09:59:17 +0100 Subject: [PATCH 002/320] More typos --- src/rdb.c | 2 +- src/server.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 8f896983b..35448b624 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1800,7 +1800,7 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) { } /* A background saving child (BGSAVE) terminated its work. Handle this. - * This function covers the case of RDB -> Salves socket transfers for + * This function covers the case of RDB -> Slaves socket transfers for * diskless replication. */ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { uint64_t *ok_slaves; diff --git a/src/server.c b/src/server.c index 85f05f1f9..94850e2ea 100644 --- a/src/server.c +++ b/src/server.c @@ -2193,7 +2193,7 @@ void preventCommandReplication(client *c) { * CMD_CALL_STATS Populate command stats. * CMD_CALL_PROPAGATE_AOF Append command to AOF if it modified the dataset * or if the client flags are forcing propagation. - * CMD_CALL_PROPAGATE_REPL Send command to salves if it modified the dataset + * CMD_CALL_PROPAGATE_REPL Send command to slaves if it modified the dataset * or if the client flags are forcing propagation. * CMD_CALL_PROPAGATE Alias for PROPAGATE_AOF|PROPAGATE_REPL. * CMD_CALL_FULL Alias for SLOWLOG|STATS|PROPAGATE. From 2eb05fe3de7b82e8f56f185ad8ab1471fe61053b Mon Sep 17 00:00:00 2001 From: jem Date: Tue, 18 Sep 2018 17:04:00 +0800 Subject: [PATCH 003/320] update leap year comment when div by 400 --- src/localtime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/localtime.c b/src/localtime.c index 3f59a3331..e2ac81f98 100644 --- a/src/localtime.c +++ b/src/localtime.c @@ -52,8 +52,8 @@ static int is_leap_year(time_t year) { if (year % 4) return 0; /* A year not divisible by 4 is not leap. */ else if (year % 100) return 1; /* If div by 4 and not 100 is surely leap. */ - else if (year % 400) return 0; /* If div by 100 *and* 400 is not leap. */ - else return 1; /* If div by 100 and not by 400 is leap. */ + else if (year % 400) return 0; /* If div by 100 *and* not by 400 is not leap. */ + else return 1; /* If div by 100 and 400 is leap. */ } void nolocks_localtime(struct tm *tmp, time_t t, time_t tz, int dst) { From 7c244182742d822ed909f47d4b4a534c49f3f862 Mon Sep 17 00:00:00 2001 From: jem Date: Tue, 18 Sep 2018 20:42:09 +0800 Subject: [PATCH 004/320] ignore vscode conf dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a188cfc82..717bf3c7c 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ deps/lua/src/liblua.a .prerequisites *.dSYM Makefile.dep +.vscode/* From b4f77cc43aefb0710fc11d3103c2fa094d10e029 Mon Sep 17 00:00:00 2001 From: Jim Brunner Date: Wed, 13 Mar 2019 16:31:24 +0000 Subject: [PATCH 005/320] Addition of OnUnload function --- src/module.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/module.c b/src/module.c index 5ad999751..ba38ed8d4 100644 --- a/src/module.c +++ b/src/module.c @@ -4799,6 +4799,23 @@ int moduleUnload(sds name) { errno = EBUSY; return REDISMODULE_ERR; } + + /* Give module a chance to clean up. */ + int (*onunload)(void *); + onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload"); + if (onunload) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = module; + ctx.client = moduleFreeContextReusedClient; + int unload_status = onunload((void*)&ctx); + moduleFreeContext(&ctx); + + if (unload_status == REDISMODULE_ERR) { + serverLog(LL_WARNING, "Module %s OnUnload failed. Unload canceled.", name); + errno = ECANCELED; + return REDISMODULE_ERR; + } + } moduleUnregisterCommands(module); From 3137f26d4c6fe0ca6e9ea4dc9e31eac89ec548e9 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 3 Jun 2018 15:37:48 +0300 Subject: [PATCH 006/320] Add RedisModule_Assert() API call. --- src/module.c | 10 ++++++++++ src/redismodule.h | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/module.c b/src/module.c index 8954fcdf0..c6e6e5989 100644 --- a/src/module.c +++ b/src/module.c @@ -3461,6 +3461,15 @@ void RM_LogIOError(RedisModuleIO *io, const char *levelstr, const char *fmt, ... va_end(ap); } +/* Redis-like assert function. + * + * A failed assertion will shut down the server and produce logging information + * that looks identical to information generated by Redis itself. + */ +void RM__Assert(const char *estr, const char *file, int line) { + _serverAssert(estr, file, line); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -4993,6 +5002,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); + REGISTER_API(_Assert); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); diff --git a/src/redismodule.h b/src/redismodule.h index d18c38881..db32df049 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -271,6 +271,7 @@ void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value) float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); +void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str); int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); @@ -433,6 +434,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(EmitAOF); REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); + REDISMODULE_GET_API(_Assert); REDISMODULE_GET_API(StringAppendBuffer); REDISMODULE_GET_API(RetainString); REDISMODULE_GET_API(StringCompare); @@ -499,6 +501,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int return REDISMODULE_OK; } +#define RedisModule_Assert(_e) ((_e)?(void)0 : (RedisModule__Assert(#_e,__FILE__,__LINE__),exit(1))) + #else /* Things only defined for the modules core, not exported to modules From 81d3e1d354e1f6cc37f880c862969f407aebebb6 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Fri, 15 Mar 2019 21:09:59 +0200 Subject: [PATCH 007/320] add CI --- .circleci/config.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..52ff69721 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,21 @@ +version: 2 +jobs: + build: + docker: + - image: circleci/buildpack-deps + steps: + - checkout + - run: + name: dep + command: sudo apt-get install -y tcl + - run: + name: Build + command: make + - run: + name: Test + command: make test +workflows: + version: 2 + workflow: + jobs: + - build From 366fe793358940b9e8b92dd5e778c73891989906 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Fri, 15 Mar 2019 21:14:15 +0200 Subject: [PATCH 008/320] add pull app --- .github/pull.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/pull.yml diff --git a/.github/pull.yml b/.github/pull.yml new file mode 100644 index 000000000..4cc55b0f5 --- /dev/null +++ b/.github/pull.yml @@ -0,0 +1,5 @@ +version: "1" +rules: + - base: unstable + upstream: antirez:unstable + mergeMethod: merge From c9cdf67d91022cec714de34b0d0e2f60506c41dd Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Fri, 15 Mar 2019 21:30:09 +0200 Subject: [PATCH 009/320] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 52ff69721..445031a35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: steps: - checkout - run: - name: dep + name: Install dependency command: sudo apt-get install -y tcl - run: name: Build From ebdcb1618fe37a06e551cb117f93f410c854678d Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Fri, 15 Mar 2019 21:35:24 +0200 Subject: [PATCH 010/320] Update pull.yml --- .github/pull.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/pull.yml b/.github/pull.yml index 4cc55b0f5..6ccf19ca9 100644 --- a/.github/pull.yml +++ b/.github/pull.yml @@ -3,3 +3,12 @@ rules: - base: unstable upstream: antirez:unstable mergeMethod: merge + - base: 5.0 + upstream: antirez:5.0 + mergeMethod: merge + - base: 4.0 + upstream: antirez:4.0 + mergeMethod: merge + - base: 3.2 + upstream: antirez:3.2 + mergeMethod: merge From 661b5097e90edc5fe2c4d08810c8190541139769 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Fri, 15 Mar 2019 22:22:06 +0200 Subject: [PATCH 011/320] Update config.yml --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 445031a35..fe2a6ea07 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,8 +14,3 @@ jobs: - run: name: Test command: make test -workflows: - version: 2 - workflow: - jobs: - - build From 971d9ee0aa7eb6463f2d95d10c8ecde9a7e3221c Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Tue, 16 Apr 2019 22:16:12 +0300 Subject: [PATCH 012/320] Adds a "Modules" section to `INFO` Fixes #6012. As long as "INFO is broken", this should be adequate IMO. Once we rework `INFO`, perhaps into RESP3, this implementation should be revisited. --- src/module.c | 19 +++++++++++++++++++ src/server.c | 7 +++++++ src/server.h | 1 + 3 files changed, 27 insertions(+) diff --git a/src/module.c b/src/module.c index c29521670..60c9a0464 100644 --- a/src/module.c +++ b/src/module.c @@ -5244,6 +5244,25 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper function for the INFO command: adds loaded modules as to info's + * output. + * + * After the call, the passed sds info string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds genModulesInfoString(sds info) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + sds name = dictGetKey(de); + struct RedisModule *module = dictGetVal(de); + + info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + } + dictReleaseIterator(di); + return info; +} + /* Redis MODULE command. * * MODULE LOAD [args...] */ diff --git a/src/server.c b/src/server.c index fb5d679cd..49a65ef57 100644 --- a/src/server.c +++ b/src/server.c @@ -4291,6 +4291,13 @@ sds genRedisInfoString(char *section) { (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); } + /* Modules */ + if (allsections || defsections || !strcasecmp(section,"modules")) { + if (sections++) info = sdscat(info,"\r\n"); + info = sdscatprintf(info,"# Modules\r\n"); + info = genModulesInfoString(info); + } + /* Command statistics */ if (allsections || !strcasecmp(section,"commandstats")) { if (sections++) info = sdscat(info,"\r\n"); diff --git a/src/server.h b/src/server.h index dfd9f7698..d832c6465 100644 --- a/src/server.h +++ b/src/server.h @@ -2268,6 +2268,7 @@ void bugReportStart(void); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); sds genRedisInfoString(char *section); +sds genModulesInfoString(sds info); void enableWatchdog(int period); void disableWatchdog(void); void watchdogScheduleSignal(int period); From 4620bfb47ad68de4b9a89c48c3978b6ad538b9ed Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Wed, 29 May 2019 14:21:47 +0800 Subject: [PATCH 013/320] aof: fix assignment for aof_fsync_offset Signed-off-by: Yuan Zhou --- src/aof.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 4744847d2..c8fb8e8f6 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1768,7 +1768,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { server.aof_selected_db = -1; /* Make sure SELECT is re-issued */ aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; - server.aof_current_size = server.aof_current_size; + server.aof_fsync_offset = server.aof_current_size; /* Clear regular AOF buffer since its contents was just written to * the new AOF from the background rewrite buffer. */ From 0e0756659128102952c3213a9b4e7161dcc9ca3d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 30 May 2019 11:51:58 +0300 Subject: [PATCH 014/320] Jemalloc: Avoid blocking on background thread lock for stats. Background threads may run for a long time, especially when the # of dirty pages is high. Avoid blocking stats calls because of this (which may cause latency spikes). see https://github.com/jemalloc/jemalloc/issues/1502 cherry picked from commit 1a71533511027dbe3f9d989659efeec446915d6b --- deps/jemalloc/src/background_thread.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deps/jemalloc/src/background_thread.c b/deps/jemalloc/src/background_thread.c index 3517a3bb8..457669c9e 100644 --- a/deps/jemalloc/src/background_thread.c +++ b/deps/jemalloc/src/background_thread.c @@ -787,7 +787,13 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { nstime_init(&stats->run_interval, 0); for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsdn, &info->mtx); + if (malloc_mutex_trylock(tsdn, &info->mtx)) { + /* + * Each background thread run may take a long time; + * avoid waiting on the stats if the thread is active. + */ + continue; + } if (info->state != background_thread_stopped) { num_runs += info->tot_n_runs; nstime_add(&stats->run_interval, &info->tot_sleep_time); From f7833d560d80604445ad34ad2ef9d829d0aef7d6 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 30 May 2019 12:51:32 +0300 Subject: [PATCH 015/320] make redis purge jemalloc after flush, and enable background purging thread jemalloc 5 doesn't immediately release memory back to the OS, instead there's a decaying mechanism, which doesn't work when there's no traffic (no allocations). this is most evident if there's no traffic after flushdb, the RSS will remain high. 1) enable jemalloc background purging 2) explicitly purge in flushdb --- src/config.c | 9 ++++++++ src/db.c | 14 ++++++++++++ src/debug.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/server.c | 2 ++ src/server.h | 1 + src/zmalloc.c | 34 ++++++++++++++++++++++++++++ src/zmalloc.h | 2 ++ 7 files changed, 124 insertions(+) diff --git a/src/config.c b/src/config.c index 7f0e9af89..16850a1fc 100644 --- a/src/config.c +++ b/src/config.c @@ -474,6 +474,10 @@ void loadServerConfigFromString(char *config) { err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; #endif } + } else if (!strcasecmp(argv[0],"jemalloc-bg-thread") && argc == 2) { + if ((server.jemalloc_bg_thread = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"daemonize") && argc == 2) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -1152,6 +1156,9 @@ void configSetCommand(client *c) { return; } #endif + } config_set_bool_field( + "jemalloc-bg-thread",server.jemalloc_bg_thread) { + set_jemalloc_bg_thread(server.jemalloc_bg_thread); } config_set_bool_field( "protected-mode",server.protected_mode) { } config_set_bool_field( @@ -1487,6 +1494,7 @@ void configGetCommand(client *c) { config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); config_get_bool_field("activedefrag", server.active_defrag_enabled); + config_get_bool_field("jemalloc-bg-thread", server.jemalloc_bg_thread); config_get_bool_field("protected-mode", server.protected_mode); config_get_bool_field("gopher-enabled", server.gopher_enabled); config_get_bool_field("io-threads-do-reads", server.io_threads_do_reads); @@ -2318,6 +2326,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,CONFIG_DEFAULT_ACTIVE_REHASHING); rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); + rewriteConfigYesNoOption(state,"jemalloc-bg-thread",server.jemalloc_bg_thread,1); rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigYesNoOption(state,"gopher-enabled",server.gopher_enabled,CONFIG_DEFAULT_GOPHER_ENABLED); rewriteConfigYesNoOption(state,"io-threads-do-reads",server.io_threads_do_reads,CONFIG_DEFAULT_IO_THREADS_DO_READS); diff --git a/src/db.c b/src/db.c index b537a29a4..50e23d6b2 100644 --- a/src/db.c +++ b/src/db.c @@ -441,6 +441,13 @@ void flushdbCommand(client *c) { signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* FLUSHALL [ASYNC] @@ -464,6 +471,13 @@ void flushallCommand(client *c) { server.dirty = saved_dirty; } server.dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif } /* This command implements DEL and LAZYDEL. */ diff --git a/src/debug.c b/src/debug.c index 0c6b5630c..c82c99b1f 100644 --- a/src/debug.c +++ b/src/debug.c @@ -297,6 +297,56 @@ void computeDatasetDigest(unsigned char *final) { } } +#ifdef USE_JEMALLOC +void mallctl_int(client *c, robj **argv, int argc) { + int ret; + /* start with the biggest size (int64), and if that fails, try smaller sizes (int32, bool) */ + int64_t old = 0, val; + if (argc > 1) { + long long ll; + if (getLongLongFromObjectOrReply(c, argv[1], &ll, NULL) != C_OK) + return; + val = ll; + } + size_t sz = sizeof(old); + while (sz > 0) { + if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, argc > 1? &val: NULL, argc > 1?sz: 0))) { + if (ret==EINVAL) { + /* size might be wrong, try a smaller one */ + sz /= 2; +#if BYTE_ORDER == BIG_ENDIAN + val <<= 8*sz; +#endif + continue; + } + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } else { +#if BYTE_ORDER == BIG_ENDIAN + old >>= 64 - 8*sz; +#endif + addReplyLongLong(c, old); + return; + } + } + addReplyErrorFormat(c,"%s", strerror(EINVAL)); +} + +void mallctl_string(client *c, robj **argv, int argc) { + int ret; + char *old; + size_t sz = sizeof(old); + /* for strings, it seems we need to first get the old value, before overriding it. */ + if ((ret=je_mallctl(argv[0]->ptr, &old, &sz, NULL, 0))) { + addReplyErrorFormat(c,"%s", strerror(ret)); + return; + } + addReplyBulkCString(c, old); + if(argc > 1) + je_mallctl(argv[0]->ptr, NULL, 0, &argv[1]->ptr, sizeof(char*)); +} +#endif + void debugCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { @@ -323,6 +373,10 @@ void debugCommand(client *c) { "STRUCTSIZE -- Return the size of different Redis core C structures.", "ZIPLIST -- Show low level info about the ziplist encoding.", "STRINGMATCH-TEST -- Run a fuzz tester against the stringmatchlen() function.", +#ifdef USE_JEMALLOC +"MALLCTL [] -- Get or set a malloc tunning integer.", +"MALLCTL-STR [] -- Get or set a malloc tunning string.", +#endif NULL }; addReplyHelp(c, help); @@ -676,6 +730,14 @@ NULL { stringmatchlen_fuzz_test(); addReplyStatus(c,"Apparently Redis did not crash: test passed"); +#ifdef USE_JEMALLOC + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl") && c->argc >= 3) { + mallctl_int(c, c->argv+2, c->argc-2); + return; + } else if(!strcasecmp(c->argv[1]->ptr,"mallctl-str") && c->argc >= 3) { + mallctl_string(c, c->argv+2, c->argc-2); + return; +#endif } else { addReplySubcommandSyntaxError(c); return; diff --git a/src/server.c b/src/server.c index 4b87b6ac2..fa2c7b1ee 100644 --- a/src/server.c +++ b/src/server.c @@ -2230,6 +2230,7 @@ void initServerConfig(void) { server.maxidletime = CONFIG_DEFAULT_CLIENT_TIMEOUT; server.tcpkeepalive = CONFIG_DEFAULT_TCP_KEEPALIVE; server.active_expire_enabled = 1; + server.jemalloc_bg_thread = 1; server.active_defrag_enabled = CONFIG_DEFAULT_ACTIVE_DEFRAG; server.active_defrag_ignore_bytes = CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES; server.active_defrag_threshold_lower = CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER; @@ -2866,6 +2867,7 @@ void initServer(void) { latencyMonitorInit(); bioInit(); initThreadedIO(); + set_jemalloc_bg_thread(server.jemalloc_bg_thread); server.initial_memory_usage = zmalloc_used_memory(); } diff --git a/src/server.h b/src/server.h index 0813f8bd1..4ae079ff2 100644 --- a/src/server.h +++ b/src/server.h @@ -1129,6 +1129,7 @@ struct redisServer { int tcpkeepalive; /* Set SO_KEEPALIVE if non-zero. */ int active_expire_enabled; /* Can be disabled for testing purposes. */ int active_defrag_enabled; + int jemalloc_bg_thread; /* Enable jemalloc background thread */ size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */ int active_defrag_threshold_lower; /* minimum percentage of fragmentation to start active defrag */ int active_defrag_threshold_upper; /* maximum percentage of fragmentation at which we use maximum effort */ diff --git a/src/zmalloc.c b/src/zmalloc.c index 5e6010278..58896a727 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -306,6 +306,7 @@ size_t zmalloc_get_rss(void) { #endif #if defined(USE_JEMALLOC) + int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident) { @@ -327,13 +328,46 @@ int zmalloc_get_allocator_info(size_t *allocated, je_mallctl("stats.allocated", allocated, &sz, NULL, 0); return 1; } + +void set_jemalloc_bg_thread(int enable) { + /* let jemalloc do purging asynchronously, required when there's no traffic + * after flushdb */ + if (enable) { + char val = 1; + je_mallctl("background_thread", NULL, 0, &val, 1); + } +} + +int jemalloc_purge() { + /* return all unused (reserved) pages to the OS */ + char tmp[32]; + unsigned narenas = 0; + size_t sz = sizeof(unsigned); + if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) { + sprintf(tmp, "arena.%d.purge", narenas); + if (!je_mallctl(tmp, NULL, 0, NULL, 0)) + return 0; + } + return -1; +} + #else + int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident) { *allocated = *resident = *active = 0; return 1; } + +void set_jemalloc_bg_thread(int enable) { + ((void)(enable)); +} + +int jemalloc_purge() { + return 0; +} + #endif /* Get the sum of the specified field (converted form kb to bytes) in diff --git a/src/zmalloc.h b/src/zmalloc.h index 6fb19b046..b136a910d 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -86,6 +86,8 @@ size_t zmalloc_used_memory(void); void zmalloc_set_oom_handler(void (*oom_handler)(size_t)); size_t zmalloc_get_rss(void); int zmalloc_get_allocator_info(size_t *allocated, size_t *active, size_t *resident); +void set_jemalloc_bg_thread(int enable); +int jemalloc_purge(); size_t zmalloc_get_private_dirty(long pid); size_t zmalloc_get_smap_bytes_by_field(char *field, long pid); size_t zmalloc_get_memory_size(void); From e0387b30f2b3203b64c4b9074d6ae78db4eb165d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 17 Jul 2019 15:08:18 +0300 Subject: [PATCH 016/320] RM_Log - add support for logging without a context or context without module for instance detached thread safe contexts, or various callbacks that don't provide a context. --- src/module.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index f4f753c00..442499b7c 100644 --- a/src/module.c +++ b/src/module.c @@ -3515,7 +3515,7 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li if (level < server.verbosity) return; - name_len = snprintf(msg, sizeof(msg),"<%s> ", module->name); + name_len = snprintf(msg, sizeof(msg),"<%s> ", module? module->name: "module"); vsnprintf(msg + name_len, sizeof(msg) - name_len, fmt, ap); serverLogRaw(level,msg); } @@ -3533,13 +3533,15 @@ void RM_LogRaw(RedisModule *module, const char *levelstr, const char *fmt, va_li * There is a fixed limit to the length of the log line this function is able * to emit, this limit is not specified but is guaranteed to be more than * a few lines of text. + * + * The ctx argument may be NULL if cannot be provided in the context of the + * caller for instance threads or callbacks, in which case a generic "module" + * will be used instead of the module name. */ void RM_Log(RedisModuleCtx *ctx, const char *levelstr, const char *fmt, ...) { - if (!ctx->module) return; /* Can only log if module is initialized */ - va_list ap; va_start(ap, fmt); - RM_LogRaw(ctx->module,levelstr,fmt,ap); + RM_LogRaw(ctx? ctx->module: NULL,levelstr,fmt,ap); va_end(ap); } From e70fbad8023e9766dd8580c25aad83a2b0a31be4 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 17 Jul 2019 08:51:02 +0300 Subject: [PATCH 017/320] Module API for Forking * create module API for forking child processes. * refactor duplicate code around creating and tracking forks by AOF and RDB. * child processes listen to SIGUSR1 and dies exitFromChild in order to eliminate a valgrind warning of unhandled signal. * note that BGSAVE error reply has changed. valgrind error is: Process terminating with default action of signal 10 (SIGUSR1) --- runtest-moduleapi | 3 +- src/aof.c | 35 +++--------- src/childinfo.c | 2 + src/db.c | 5 +- src/defrag.c | 2 +- src/module.c | 100 ++++++++++++++++++++++++++++++++++ src/rdb.c | 49 +++-------------- src/redismodule.h | 7 +++ src/replication.c | 6 +- src/scripting.c | 3 +- src/server.c | 90 ++++++++++++++++++++++++++---- src/server.h | 9 +++ tests/modules/Makefile | 6 +- tests/modules/fork.c | 84 ++++++++++++++++++++++++++++ tests/unit/moduleapi/fork.tcl | 32 +++++++++++ 15 files changed, 342 insertions(+), 91 deletions(-) create mode 100644 tests/modules/fork.c create mode 100644 tests/unit/moduleapi/fork.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..da61ab25c 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,5 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork "${@}" + diff --git a/src/aof.c b/src/aof.c index 565ee8073..fc62d86ed 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,9 +264,9 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (server.rdb_child_pid != -1) { + if (hasForkChild() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; - serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible."); + serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { /* If there is a pending AOF rewrite, we need to switch it off and * start a new one: the old one cannot be reused because it is not @@ -397,7 +397,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) { + } else if (hasForkChild()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -493,9 +493,8 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && - (server.aof_child_pid != -1 || server.rdb_child_pid != -1)) - return; + if (server.aof_no_fsync_on_rewrite && hasForkChild()) + return; /* Perform the fsync if needed. */ if (server.aof_fsync == AOF_FSYNC_ALWAYS) { @@ -1562,39 +1561,24 @@ void aofClosePipes(void) { */ int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { char tmpfile[256]; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "AOF rewrite: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_AOF); + sendChildCOWInfo(CHILD_INFO_TYPE_AOF, "AOF rewrite"); exitFromChild(0); } else { exitFromChild(1); } } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); serverLog(LL_WARNING, @@ -1608,7 +1592,6 @@ int rewriteAppendOnlyFileBackground(void) { server.aof_rewrite_scheduled = 0; server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; - updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the * feedAppendOnlyFile() to issue a SELECT command, so the differences * accumulated by the parent into server.aof_rewrite_buf will start @@ -1623,7 +1606,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (server.rdb_child_pid != -1) { + } else if (hasForkChild()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/childinfo.c b/src/childinfo.c index 719025e8c..fa0600552 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -80,6 +80,8 @@ void receiveChildInfo(void) { server.stat_rdb_cow_bytes = server.child_info_data.cow_size; } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_AOF) { server.stat_aof_cow_bytes = server.child_info_data.cow_size; + } else if (server.child_info_data.process_type == CHILD_INFO_TYPE_MODULE) { + server.stat_module_cow_bytes = server.child_info_data.cow_size; } } } diff --git a/src/db.c b/src/db.c index 51f5a12b4..4a489036a 100644 --- a/src/db.c +++ b/src/db.c @@ -60,10 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && - !(flags & LOOKUP_NOTOUCH)) - { + if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index ecf0255dc..93c6a4619 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (server.aof_child_pid!=-1 || server.rdb_child_pid!=-1) + if (hasForkChild()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index f4f753c00..03cc36702 100644 --- a/src/module.c +++ b/src/module.c @@ -30,6 +30,7 @@ #include "server.h" #include "cluster.h" #include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" @@ -291,6 +292,14 @@ typedef struct RedisModuleCommandFilter { /* Registered filters */ static list *moduleCommandFilters; +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); + +static struct RedisModuleForkInfo { + RedisModuleForkDoneHandler done_handler; + void* done_handler_user_data; +} moduleForkInfo = {0}; + + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -5028,6 +5037,94 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* -------------------------------------------------------------------------- + * Module fork API + * -------------------------------------------------------------------------- */ + +/* Create a background child process with the current frozen snaphost of the + * main process where you can do some processing in the background without + * affecting / freezing the traffic and no need for threads and GIL locking. + * Note that Redis allows for only one concurrent fork. + * When the child wants to exit, it should call RedisModule_ExitFromChild. + * If the parent wants to kill the child it should call RedisModule_KillForkChild + * The done handler callback will be executed on the parent process when the + * child existed (but not when killed) + * Return: -1 on failure, on success the parent process will get a positive PID + * of the child, and the child process will get 0. + */ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) +{ + pid_t childpid; + if (hasForkChild()) { + return -1; + } + + openChildInfoPipe(); + if ((childpid = redisFork()) == 0) { + /* Child */ + redisSetProcTitle("redis-module-fork"); + } else if (childpid == -1) { + closeChildInfoPipe(); + serverLog(LL_WARNING,"Can't fork for module: %s", strerror(errno)); + } else { + /* Parent */ + server.module_child_pid = childpid; + moduleForkInfo.done_handler = cb; + moduleForkInfo.done_handler_user_data = user_data; + serverLog(LL_NOTICE, "Module fork started pid: %d ", childpid); + } + return childpid; +} + +/* Call from the child process when you want to terminate it. + * retcode will be provided to the done handler executed on the parent process. + */ +int RM_ExitFromChild(int retcode) +{ + sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); + exitFromChild(retcode); + return REDISMODULE_OK; +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + int statloc; + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + serverLog(LL_NOTICE,"Killing running module fork child: %ld", + (long) server.module_child_pid); + if (kill(server.module_child_pid,SIGUSR1) != -1) { + while(wait3(&statloc,0,NULL) != server.module_child_pid); + } + /* Reset the buffer accumulating changes while the child saves. */ + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; + closeChildInfoPipe(); + updateDictResizePolicy(); + return REDISMODULE_OK; +} + +void ModuleForkDoneHandler(int exitcode, int bysignal) +{ + serverLog(LL_NOTICE, + "Module fork exited pid: %d, retcode: %d, bysignal: %d", + server.module_child_pid, exitcode, bysignal); + if (moduleForkInfo.done_handler) { + moduleForkInfo.done_handler(exitcode, bysignal, + moduleForkInfo.done_handler_user_data); + } + server.module_child_pid = -1; + moduleForkInfo.done_handler = NULL; + moduleForkInfo.done_handler_user_data = NULL; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5490,4 +5587,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(Fork); + REGISTER_API(ExitFromChild); + REGISTER_API(KillForkChild); } diff --git a/src/rdb.c b/src/rdb.c index c566378fb..0c3a80d01 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1293,40 +1293,25 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - long long start; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { int retval; /* Child */ - closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename,rsi); if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); } exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); if (childpid == -1) { closeChildInfoPipe(); server.lastbgsave_status = C_ERR; @@ -1338,7 +1323,6 @@ int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_DISK; - updateDictResizePolicy(); return C_OK; } return C_OK; /* unreached */ @@ -2279,10 +2263,9 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { listNode *ln; listIter li; pid_t childpid; - long long start; int pipefds[2]; - if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; + if (hasForkChild()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2318,8 +2301,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { /* Create the child process. */ openChildInfoPipe(); - start = ustime(); - if ((childpid = fork()) == 0) { + if ((childpid = redisFork()) == 0) { /* Child */ int retval; rio slave_sockets; @@ -2327,7 +2309,6 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { rioInitWithFdset(&slave_sockets,fds,numfds); zfree(fds); - closeListeningSockets(0); redisSetProcTitle("redis-rdb-to-slaves"); retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); @@ -2335,16 +2316,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { retval = C_ERR; if (retval == C_OK) { - size_t private_dirty = zmalloc_get_private_dirty(-1); - - if (private_dirty) { - serverLog(LL_NOTICE, - "RDB: %zu MB of memory used by copy-on-write", - private_dirty/(1024*1024)); - } - - server.child_info_data.cow_size = private_dirty; - sendChildInfo(CHILD_INFO_TYPE_RDB); + sendChildCOWInfo(CHILD_INFO_TYPE_RDB, "RDB"); /* If we are returning OK, at least one slave was served * with the RDB file as expected, so we need to send a report @@ -2413,16 +2385,11 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { close(pipefds[1]); closeChildInfoPipe(); } else { - server.stat_fork_time = ustime()-start; - server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ - latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); - serverLog(LL_NOTICE,"Background RDB transfer started by pid %d", childpid); server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; - updateDictResizePolicy(); } zfree(clientids); zfree(fds); @@ -2465,13 +2432,13 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (server.aof_child_pid != -1) { + } else if (hasForkChild()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "An AOF log rewriting in progress: can't BGSAVE right now. " + "Another BG operation is in progress: can't BGSAVE right now. " "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " "possible."); } diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..60681da7c 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -173,6 +173,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value); typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); +typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); #define REDISMODULE_TYPE_METHOD_VERSION 1 typedef struct RedisModuleTypeMethods { @@ -357,6 +358,9 @@ const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CommandFilterArgGet)(R int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgInsert)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgReplace)(RedisModuleCommandFilterCtx *fctx, int pos, RedisModuleString *arg); int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandFilterCtx *fctx, int pos); +int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); +int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); +int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); #endif /* This is included inline inside each Redis module. */ @@ -528,6 +532,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CommandFilterArgInsert); REDISMODULE_GET_API(CommandFilterArgReplace); REDISMODULE_GET_API(CommandFilterArgDelete); + REDISMODULE_GET_API(Fork); + REDISMODULE_GET_API(ExitFromChild); + REDISMODULE_GET_API(KillForkChild); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..7adf5ba38 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,11 +751,11 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (server.aof_child_pid == -1) { + if (!hasForkChild()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, - "No BGSAVE in progress, but an AOF rewrite is active. " + "No BGSAVE in progress, but another BG operation is active. " "BGSAVE for replication delayed"); } } @@ -2899,7 +2899,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/scripting.c b/src/scripting.c index 032bfdf10..deb406457 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1693,7 +1693,7 @@ void ldbSendLogs(void) { int ldbStartSession(client *c) { ldb.forked = (c->flags & CLIENT_LUA_DEBUG_SYNC) == 0; if (ldb.forked) { - pid_t cp = fork(); + pid_t cp = redisFork(); if (cp == -1) { addReplyError(c,"Fork() failed: can't run EVAL in debugging mode."); return 0; @@ -1710,7 +1710,6 @@ int ldbStartSession(client *c) { * socket to make sure if the parent crashes a reset is sent * to the clients. */ serverLog(LL_WARNING,"Redis forked for debugging eval"); - closeListeningSockets(0); } else { /* Parent */ listAddNodeTail(ldb.children,(void*)(unsigned long)cp); diff --git a/src/server.c b/src/server.c index 4337b8f01..675b638d6 100644 --- a/src/server.c +++ b/src/server.c @@ -1447,12 +1447,18 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) + if (!hasForkChild()) dictEnableResize(); else dictDisableResize(); } +int hasForkChild() { + return server.rdb_child_pid != -1 || + server.aof_child_pid != -1 || + server.module_child_pid != -1; +} + /* ======================= Cron: called every 100 ms ======================== */ /* Add a sample to the operations per second array of samples. */ @@ -1689,7 +1695,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + if (!hasForkChild()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1886,15 +1892,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (server.rdb_child_pid != -1 || server.aof_child_pid != -1 || - ldbPendingChildren()) + if (hasForkChild() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1907,16 +1912,20 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " - "rdb_child_pid = %d, aof_child_pid = %d", + "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", strerror(errno), (int) server.rdb_child_pid, - (int) server.aof_child_pid); + (int) server.aof_child_pid, + (int) server.module_child_pid); } else if (pid == server.rdb_child_pid) { backgroundSaveDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); } else if (pid == server.aof_child_pid) { backgroundRewriteDoneHandler(exitcode,bysignal); if (!bysignal && exitcode == 0) receiveChildInfo(); + } else if (pid == server.module_child_pid) { + ModuleForkDoneHandler(exitcode,bysignal); + if (!bysignal && exitcode == 0) receiveChildInfo(); } else { if (!ldbRemoveChild(pid)) { serverLog(LL_WARNING, @@ -1954,8 +1963,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - server.rdb_child_pid == -1 && - server.aof_child_pid == -1 && + !hasForkChild() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2013,7 +2021,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && + if (!hasForkChild() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) @@ -2784,6 +2792,7 @@ void initServer(void) { server.cronloops = 0; server.rdb_child_pid = -1; server.aof_child_pid = -1; + server.module_child_pid = -1; server.rdb_child_type = RDB_CHILD_TYPE_NONE; server.rdb_bgsave_scheduled = 0; server.child_info_pipe[0] = -1; @@ -2802,6 +2811,7 @@ void initServer(void) { server.stat_peak_memory = 0; server.stat_rdb_cow_bytes = 0; server.stat_aof_cow_bytes = 0; + server.stat_module_cow_bytes = 0; server.cron_malloc_stats.zmalloc_used = 0; server.cron_malloc_stats.process_rss = 0; server.cron_malloc_stats.allocator_allocated = 0; @@ -4040,7 +4050,9 @@ sds genRedisInfoString(char *section) { "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n" "aof_last_write_status:%s\r\n" - "aof_last_cow_size:%zu\r\n", + "aof_last_cow_size:%zu\r\n" + "module_fork_in_progress:%d\r\n" + "module_fork_last_cow_size:%zu\r\n", server.loading, server.dirty, server.rdb_child_pid != -1, @@ -4058,7 +4070,9 @@ sds genRedisInfoString(char *section) { -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == C_OK) ? "ok" : "err", (server.aof_last_write_status == C_OK) ? "ok" : "err", - server.stat_aof_cow_bytes); + server.stat_aof_cow_bytes, + server.module_child_pid != -1, + server.stat_module_cow_bytes); if (server.aof_enabled) { info = sdscatprintf(info, @@ -4554,6 +4568,58 @@ void setupSignalHandlers(void) { return; } +static void sigKillChildHandler(int sig) { + UNUSED(sig); + /* this handler is needed to resolve a valgrind warning */ + serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); + exitFromChild(1); +} + +void setupChildSignalHandlers(void) { + struct sigaction act; + + /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction is used. + * Otherwise, sa_handler is used. */ + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = sigKillChildHandler; + sigaction(SIGUSR1, &act, NULL); + return; +} + +int redisFork() { + int childpid; + long long start = ustime(); + if ((childpid = fork()) == 0) { + /* Child */ + closeListeningSockets(0); + setupChildSignalHandlers(); + } else { + /* Parent */ + server.stat_fork_time = ustime()-start; + server.stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / server.stat_fork_time / (1024*1024*1024); /* GB per second. */ + latencyAddSampleIfNeeded("fork",server.stat_fork_time/1000); + if (childpid == -1) { + return -1; + } + updateDictResizePolicy(); + } + return childpid; +} + +void sendChildCOWInfo(int ptype, char *pname) { + size_t private_dirty = zmalloc_get_private_dirty(-1); + + if (private_dirty) { + serverLog(LL_NOTICE, + "%s: %zu MB of memory used by copy-on-write", + pname, private_dirty/(1024*1024)); + } + + server.child_info_data.cow_size = private_dirty; + sendChildInfo(ptype); +} + void memtest(size_t megabytes, int passes); /* Returns 1 if there is --sentinel among the arguments or if diff --git a/src/server.h b/src/server.h index f81b1010e..deaaf2636 100644 --- a/src/server.h +++ b/src/server.h @@ -1030,6 +1030,7 @@ struct clusterState; #define CHILD_INFO_MAGIC 0xC17DDA7A12345678LL #define CHILD_INFO_TYPE_RDB 0 #define CHILD_INFO_TYPE_AOF 1 +#define CHILD_INFO_TYPE_MODULE 3 struct redisServer { /* General */ @@ -1065,6 +1066,7 @@ struct redisServer { int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a client blocked on a module command needs to be processed. */ + pid_t module_child_pid; /* PID of module child */ /* Networking */ int port; /* TCP listening port */ int tcp_backlog; /* TCP listen() backlog */ @@ -1138,6 +1140,7 @@ struct redisServer { _Atomic long long stat_net_output_bytes; /* Bytes written to network. */ size_t stat_rdb_cow_bytes; /* Copy on write bytes during RDB saving. */ size_t stat_aof_cow_bytes; /* Copy on write bytes during AOF rewrite. */ + size_t stat_module_cow_bytes; /* Copy on write bytes during module fork. */ /* The following two are used to track instantaneous metrics, like * number of operations per second, network traffic. */ struct { @@ -1528,6 +1531,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +void ModuleForkDoneHandler(int exitcode, int bysignal); /* Utils */ long long ustime(void); @@ -1786,6 +1790,11 @@ void closeChildInfoPipe(void); void sendChildInfo(int process_type); void receiveChildInfo(void); +/* Fork helpers */ +int redisFork(); +int hasForkChild(); +void sendChildCOWInfo(int ptype, char *pname); + /* acl.c -- Authentication related prototypes. */ extern rax *Users; extern user *DefaultUser; diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20afa..846d4c87d 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,16 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so fork.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +fork.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +fork.so: fork.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/fork.c b/tests/modules/fork.c new file mode 100644 index 000000000..0804e4355 --- /dev/null +++ b/tests/modules/fork.c @@ -0,0 +1,84 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include + +#define UNUSED(V) ((void) V) + +int child_pid = -1; +int exitted_with_code = -1; + +void done_handler(int exitcode, int bysignal, void *user_data) { + child_pid = -1; + exitted_with_code = exitcode; + assert(user_data==(void*)0xdeadbeef); + UNUSED(bysignal); +} + +int fork_create(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + long long code_to_exit_with; + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModule_StringToLongLong(argv[1], &code_to_exit_with); + exitted_with_code = -1; + child_pid = RedisModule_Fork(done_handler, (void*)0xdeadbeef); + if (child_pid < 0) { + RedisModule_ReplyWithError(ctx, "Fork failed"); + return REDISMODULE_OK; + } else if (child_pid > 0) { + /* parent */ + RedisModule_ReplyWithLongLong(ctx, child_pid); + return REDISMODULE_OK; + } + + /* child */ + RedisModule_Log(ctx, "notice", "fork child started"); + usleep(200000); + RedisModule_Log(ctx, "notice", "fork child exiting"); + RedisModule_ExitFromChild(code_to_exit_with); + /* unreachable */ + return 0; +} + +int fork_exitcode(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + RedisModule_ReplyWithLongLong(ctx, exitted_with_code); + return REDISMODULE_OK; +} + +int fork_kill(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + UNUSED(argv); + UNUSED(argc); + if (RedisModule_KillForkChild(child_pid) != REDISMODULE_OK) + RedisModule_ReplyWithError(ctx, "KillForkChild failed"); + else + RedisModule_ReplyWithLongLong(ctx, 1); + child_pid = -1; + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + if (RedisModule_Init(ctx,"fork",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.create", fork_create,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.exitcode", fork_exitcode,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fork.kill", fork_kill,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/fork.tcl b/tests/unit/moduleapi/fork.tcl new file mode 100644 index 000000000..f7d7e47d5 --- /dev/null +++ b/tests/unit/moduleapi/fork.tcl @@ -0,0 +1,32 @@ +set testmodule [file normalize tests/modules/fork.so] + +proc count_log_message {pattern} { + set result [exec grep -c $pattern < [srv 0 stdout]] +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module fork} { + # the argument to fork.create is the exitcode on termination + r fork.create 3 + wait_for_condition 20 100 { + [r fork.exitcode] != -1 + } else { + fail "fork didn't terminate" + } + r fork.exitcode + } {3} + + test {Module fork kill} { + r fork.create 3 + after 20 + r fork.kill + after 100 + + assert {[count_log_message "fork child started"] eq "2"} + assert {[count_log_message "Received SIGUSR1 in child"] eq "1"} + assert {[count_log_message "fork child exiting"] eq "1"} + } + +} From 92455bb0360a0c914f89edfedb3ffd7fc4e90b72 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 19 Jul 2019 11:12:39 +0200 Subject: [PATCH 018/320] RDB: fix MODULE_AUX loading by continuing to next opcode. Thanks to @JohnSully for noticing this problem. --- src/rdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rdb.c b/src/rdb.c index 97fad2573..2118a00f4 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2116,6 +2116,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { /* RDB check mode. */ robj *aux = rdbLoadCheckModuleValue(rdb,name); decrRefCount(aux); + continue; /* Read next opcode. */ } } From c80ad2f4b53c3345f565e86fbdbcac9403ad094c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 14 Jul 2019 17:34:13 +0300 Subject: [PATCH 019/320] Allow modules to handle RDB loading errors. This is especially needed in diskless loading, were a short read could have caused redis to exit. now the module can handle the error and return to the caller gracefully. this fixes #5326 --- src/module.c | 41 +++++++++++++++++++++++++++++++++++------ src/redismodule.h | 9 +++++++++ src/server.h | 2 ++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/module.c b/src/module.c index f4f753c00..68952b1e0 100644 --- a/src/module.c +++ b/src/module.c @@ -3139,9 +3139,14 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * RDB loading and saving functions * -------------------------------------------------------------------------- */ -/* Called when there is a load error in the context of a module. This cannot - * be recovered like for the built-in types. */ +/* Called when there is a load error in the context of a module. On some + * modules this cannot be recovered, but if the module declared capability + * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { + if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + io->error = 1; + return; + } serverLog(LL_WARNING, "Error loading data from RDB (short read or EOF). " "Read performed by module '%s' about type '%s' " @@ -3152,6 +3157,23 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } +/* Set flags defining capabilities or behavior */ +void RM_SetIOFlags(RedisModuleIO *io, int flags) { + io->flags = flags; +} + +/* Get flags which were set by RedisModule_SetIOFlags */ +int RM_GetIOFlags(RedisModuleIO *io) { + return io->flags; +} + +/* Returns true if any previous IO API failed. + * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetIOFlags first. */ +int RM_IsIOError(RedisModuleIO *io) { + return io->error; +} + /* Save an unsigned 64 bit value into the RDB file. This function should only * be called in the context of the rdb_save method of modules implementing new * data types. */ @@ -3175,6 +3197,7 @@ saveerr: * be called in the context of the rdb_load method of modules implementing * new data types. */ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_UINT) goto loaderr; @@ -3186,7 +3209,7 @@ uint64_t RM_LoadUnsigned(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* Like RedisModule_SaveUnsigned() but for signed 64 bit values. */ @@ -3245,6 +3268,7 @@ saveerr: /* Implements RM_LoadString() and RM_LoadStringBuffer() */ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { + if (io->error) return NULL; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_STRING) goto loaderr; @@ -3256,7 +3280,7 @@ void *moduleLoadString(RedisModuleIO *io, int plain, size_t *lenptr) { loaderr: moduleRDBLoadError(io); - return NULL; /* Never reached. */ + return NULL; } /* In the context of the rdb_load method of a module data type, loads a string @@ -3305,6 +3329,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * double value saved by RedisModule_SaveDouble(). */ double RM_LoadDouble(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_DOUBLE) goto loaderr; @@ -3316,7 +3341,7 @@ double RM_LoadDouble(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* In the context of the rdb_save method of a module data type, saves a float @@ -3341,6 +3366,7 @@ saveerr: /* In the context of the rdb_save method of a module data type, loads back the * float value saved by RedisModule_SaveFloat(). */ float RM_LoadFloat(RedisModuleIO *io) { + if (io->error) return 0; if (io->ver == 2) { uint64_t opcode = rdbLoadLen(io->rio,NULL); if (opcode != RDB_MODULE_OPCODE_FLOAT) goto loaderr; @@ -3352,7 +3378,7 @@ float RM_LoadFloat(RedisModuleIO *io) { loaderr: moduleRDBLoadError(io); - return 0; /* Never reached. */ + return 0; } /* -------------------------------------------------------------------------- @@ -5408,6 +5434,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); + REGISTER_API(GetIOFlags); + REGISTER_API(SetIOFlags); + REGISTER_API(IsIOError); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..154b5b405 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,6 +140,9 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) +/* Declare that the module can handle errors with RedisModule_SetIOFlags */ +#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE @@ -271,6 +274,9 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); +void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); +int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); +int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -444,6 +450,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); + REDISMODULE_GET_API(SetIOFlags); + REDISMODULE_GET_API(GetIOFlags); + REDISMODULE_GET_API(IsIOError); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/server.h b/src/server.h index f81b1010e..9957c3b5c 100644 --- a/src/server.h +++ b/src/server.h @@ -599,6 +599,7 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ + int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -611,6 +612,7 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ + iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis From 4b89b52355947e2b7aba444139575a575f994fa2 Mon Sep 17 00:00:00 2001 From: chendianqiang Date: Mon, 22 Jul 2019 17:45:30 +0800 Subject: [PATCH 020/320] make memory usage consistent of robj with OBJ_ENCODING_INT --- src/object.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index ec0bd02ee..43f46b020 100644 --- a/src/object.c +++ b/src/object.c @@ -455,10 +455,15 @@ robj *tryObjectEncoding(robj *o) { incrRefCount(shared.integers[value]); return shared.integers[value]; } else { - if (o->encoding == OBJ_ENCODING_RAW) sdsfree(o->ptr); - o->encoding = OBJ_ENCODING_INT; - o->ptr = (void*) value; - return o; + if (o->encoding == OBJ_ENCODING_RAW) { + sdsfree(o->ptr); + o->encoding = OBJ_ENCODING_INT; + o->ptr = (void*) value; + return o; + } else { + decrRefCount(o); + return createStringObjectFromLongLongForValue(value); + } } } From bcf3cf8832940517fa73f30f7a35dcb5d4614943 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 11:47:44 +0200 Subject: [PATCH 021/320] Clinet side caching: take count of used caching slots. --- src/tracking.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index 84b29d28a..5938efade 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -60,6 +60,7 @@ * use the most significant bits instead of the full 24 bits. */ #define TRACKING_TABLE_SIZE (1<<24) rax **TrackingTable = NULL; +unsigned long TrackingTableUsedSlots = 0; robj *TrackingChannelName; /* Remove the tracking state from the client 'c'. Note that there is not much @@ -109,8 +110,10 @@ void trackingRememberKeys(client *c) { sds sdskey = c->argv[idx]->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) + if (TrackingTable[hash] == NULL) { TrackingTable[hash] = raxNew(); + TrackingTableUsedSlots++; + } raxTryInsert(TrackingTable[hash], (unsigned char*)&c->id,sizeof(c->id),NULL,NULL); } @@ -176,6 +179,7 @@ void trackingInvalidateKey(robj *keyobj) { * again if more keys will be modified in this hash slot. */ raxFree(TrackingTable[hash]); TrackingTable[hash] = NULL; + TrackingTableUsedSlots--; } void trackingInvalidateKeysOnFlush(int dbid) { From ab490723629cd50ab49cc699023c5af31ab427e7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:10:51 +0200 Subject: [PATCH 022/320] Client side caching: don't hash the key if not needed. --- src/tracking.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 5938efade..7ee4d0b2c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -156,12 +156,15 @@ void sendTrackingMessage(client *c, long long hash) { /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is - * to send a notification to every client that may have keys about such . */ + * to send a notification to every client that may have keys about such caching + * slot. */ void trackingInvalidateKey(robj *keyobj) { + if (TrackingTable == NULL || TrackingTableUsedSlots == 0) return; + sds sdskey = keyobj->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable == NULL || TrackingTable[hash] == NULL) return; + if (TrackingTable[hash] == NULL) return; raxIterator ri; raxStart(&ri,TrackingTable[hash]); From df2e7cf4fc93055542175207055f4c43aaa57880 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:29:54 +0200 Subject: [PATCH 023/320] Client side caching: call the invalidation functions always. Otherwise what happens is that the tracking table will never get garbage collected if there are no longer clients with tracking enabled. Now the invalidation function immediately checks if there is any table allocated, otherwise it returns ASAP, so the overhead when the feature is not used should be near zero. --- src/db.c | 4 ++-- src/expire.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 568e0b8de..cc31a117b 100644 --- a/src/db.c +++ b/src/db.c @@ -412,12 +412,12 @@ long long dbTotalServerKeyCount() { void signalModifiedKey(redisDb *db, robj *key) { touchWatchedKey(db,key); - if (server.tracking_clients) trackingInvalidateKey(key); + trackingInvalidateKey(key); } void signalFlushedDb(int dbid) { touchWatchedKeysOnFlush(dbid); - if (server.tracking_clients) trackingInvalidateKeysOnFlush(dbid); + trackingInvalidateKeysOnFlush(dbid); } /*----------------------------------------------------------------------------- diff --git a/src/expire.c b/src/expire.c index b23117a3c..598b27f96 100644 --- a/src/expire.c +++ b/src/expire.c @@ -64,7 +64,7 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { dbSyncDelete(db,keyobj); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",keyobj,db->id); - if (server.tracking_clients) trackingInvalidateKey(keyobj); + trackingInvalidateKey(keyobj); decrRefCount(keyobj); server.stat_expiredkeys++; return 1; From 06de8790dcf95d0e724e06c0f6f164c1fb2c08e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:31:46 +0200 Subject: [PATCH 024/320] Client side caching: reclaim the tracking table on FLUSHALL. --- src/tracking.c | 52 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 7ee4d0b2c..dc2934831 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -185,17 +185,53 @@ void trackingInvalidateKey(robj *keyobj) { TrackingTableUsedSlots--; } +/* This function is called when one or all the Redis databases are flushed + * (dbid == -1 in case of FLUSHALL). Caching slots are not specific for + * each DB but are global: currently what we do is sending a special + * notification to clients with tracking enabled, invalidating the caching + * slot "-1", which means, "all the keys", in order to avoid flooding clients + * with many invalidation messages for all the keys they may hold. + * + * However trying to flush the tracking table here is very costly: + * we need scanning 16 million caching slots in the table to check + * if they are used, this introduces a big delay. So what we do is to really + * flush the table in the case of FLUSHALL. When a FLUSHDB is called instead + * we just send the invalidation message to all the clients, but don't + * flush the table: it will slowly get garbage collected as more keys + * are modified in the used caching slots. */ void trackingInvalidateKeysOnFlush(int dbid) { UNUSED(dbid); - if (server.tracking_clients == 0) return; - listNode *ln; - listIter li; - listRewind(server.clients,&li); - while ((ln = listNext(&li)) != NULL) { - client *c = listNodeValue(ln); - if (c->flags & CLIENT_TRACKING) { - sendTrackingMessage(c,-1); + if (server.tracking_clients) { + listNode *ln; + listIter li; + listRewind(server.clients,&li); + while ((ln = listNext(&li)) != NULL) { + client *c = listNodeValue(ln); + if (c->flags & CLIENT_TRACKING) { + sendTrackingMessage(c,-1); + } + } + } + + /* In case of FLUSHALL, reclaim all the memory used by tracking. */ + if (dbid == -1 && TrackingTable) { + for (int j = 0; j < TRACKING_TABLE_SIZE; j++) { + if (TrackingTable[j] != NULL) { + raxFree(TrackingTable[j]); + TrackingTable[j] = NULL; + TrackingTableUsedSlots--; + if (TrackingTableUsedSlots == 0) break; + } + } + + /* If there are no clients with tracking enabled, we can even + * reclaim the memory used by the table itself. The code assumes + * the table is allocated only if there is at least one client alive + * with tracking enabled. */ + if (server.tracking_clients == 0) { + zfree(TrackingTable); + TrackingTable = NULL; } } } From ac05029790f88e5897275773e26cb9acb7dc2786 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 12:37:43 +0200 Subject: [PATCH 025/320] Move signalFlushedDb() into a better place. Now that the call also invalidates client side caching slots, it is important that after an internal flush operation we both send the notifications to the clients and, at the same time, are able to reclaim the memory of the tracking table. This may even fix a few edge cases related to MULTI/EXEC + WATCH during resync, not sure, but in general looks more correct. --- src/db.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index cc31a117b..3d8b844e8 100644 --- a/src/db.c +++ b/src/db.c @@ -378,6 +378,7 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( } } if (dbnum == -1) flushSlaveKeysWithExpireList(); + signalFlushedDb(dbnum); return removed; } @@ -453,7 +454,6 @@ void flushdbCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(c->db->id); server.dirty += emptyDb(c->db->id,flags,NULL); addReply(c,shared.ok); } @@ -465,7 +465,6 @@ void flushallCommand(client *c) { int flags; if (getFlushCommandFlags(c,&flags) == C_ERR) return; - signalFlushedDb(-1); server.dirty += emptyDb(-1,flags,NULL); addReply(c,shared.ok); if (server.rdb_child_pid != -1) killRDBChild(); From 023b67e80f23921da305c0c6a4615f7bb95d9049 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 18:45:47 +0200 Subject: [PATCH 026/320] Client side caching: redis-cli ability to enable tracking. This is extremely useful in order to simulate an high load of requests about different keys, and force Redis to track a lot of informations about several clients, to simulate real world workloads. --- src/redis-benchmark.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 1d16fa4ee..2df41580b 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -104,6 +104,7 @@ static struct config { int is_fetching_slots; int is_updating_slots; int slots_last_update; + int enable_tracking; /* Thread mutexes to be used as fallbacks by atomicvar.h */ pthread_mutex_t requests_issued_mutex; pthread_mutex_t requests_finished_mutex; @@ -255,7 +256,7 @@ static redisConfig *getRedisConfig(const char *ip, int port, goto fail; } - if(config.auth){ + if(config.auth) { void *authReply = NULL; redisAppendCommand(c, "AUTH %s", config.auth); if (REDIS_OK != redisGetReply(c, &authReply)) goto fail; @@ -633,6 +634,14 @@ static client createClient(char *cmd, size_t len, client from, int thread_id) { c->prefix_pending++; } + if (config.enable_tracking) { + char *buf = NULL; + int len = redisFormatCommand(&buf, "CLIENT TRACKING on"); + c->obuf = sdscatlen(c->obuf, buf, len); + free(buf); + c->prefix_pending++; + } + /* If a DB number different than zero is selected, prefix our request * buffer with the SELECT command, that will be discarded the first * time the replies are received, so if the client is reused the @@ -1350,6 +1359,8 @@ int parseOptions(int argc, const char **argv) { } else if (config.num_threads < 0) config.num_threads = 0; } else if (!strcmp(argv[i],"--cluster")) { config.cluster_mode = 1; + } else if (!strcmp(argv[i],"--enable-tracking")) { + config.enable_tracking = 1; } else if (!strcmp(argv[i],"--help")) { exit_status = 0; goto usage; @@ -1380,6 +1391,7 @@ usage: " --dbnum SELECT the specified db number (default 0)\n" " --threads Enable multi-thread mode.\n" " --cluster Enable cluster mode.\n" +" --enable-tracking Send CLIENT TRACKING on before starting benchmark.\n" " -k 1=keep alive 0=reconnect (default 1)\n" " -r Use random keys for SET/GET/INCR, random values for SADD\n" " Using this option the benchmark will expand the string __rand_int__\n" @@ -1504,6 +1516,7 @@ int main(int argc, const char **argv) { config.is_fetching_slots = 0; config.is_updating_slots = 0; config.slots_last_update = 0; + config.enable_tracking = 0; i = parseOptions(argc,argv); argc -= i; From f2f3d91e781af95f3163f2322f6b5ef364afdfd7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Jul 2019 18:59:53 +0200 Subject: [PATCH 027/320] Client side caching: split invalidation into key / slot. --- src/server.c | 4 ++++ src/server.h | 1 + src/tracking.c | 58 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/server.c b/src/server.c index 4337b8f01..5e18b5465 100644 --- a/src/server.c +++ b/src/server.c @@ -3401,6 +3401,10 @@ int processCommand(client *c) { } } + /* Make sure to use a reasonable amount of memory for client side + * caching metadata. */ + if (server.tracking_clients) trackingLimitUsedSlots(); + /* Don't accept write commands if there are problems persisting on disk * and if this is a master instance. */ int deny_write_type = writeCommandsDeniedByDiskError(); diff --git a/src/server.h b/src/server.h index b200a6696..fdab53830 100644 --- a/src/server.h +++ b/src/server.h @@ -1639,6 +1639,7 @@ void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); +void trackingLimitUsedSlots(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index dc2934831..1189e82d5 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -154,6 +154,30 @@ void sendTrackingMessage(client *c, long long hash) { } } +/* Invalidates a caching slot: this is actually the low level implementation + * of the API that Redis calls externally, that is trackingInvalidateKey(). */ +void trackingInvalidateSlot(uint64_t slot) { + if (TrackingTable == NULL || TrackingTable[slot] == NULL) return; + + raxIterator ri; + raxStart(&ri,TrackingTable[slot]); + raxSeek(&ri,"^",NULL,0); + while(raxNext(&ri)) { + uint64_t id; + memcpy(&id,ri.key,ri.key_len); + client *c = lookupClientByID(id); + if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; + sendTrackingMessage(c,slot); + } + raxStop(&ri); + + /* Free the tracking table: we'll create the radix tree and populate it + * again if more keys will be modified in this caching slot. */ + raxFree(TrackingTable[slot]); + TrackingTable[slot] = NULL; + TrackingTableUsedSlots--; +} + /* This function is called from signalModifiedKey() or other places in Redis * when a key changes value. In the context of keys tracking, our task here is * to send a notification to every client that may have keys about such caching @@ -164,25 +188,7 @@ void trackingInvalidateKey(robj *keyobj) { sds sdskey = keyobj->ptr; uint64_t hash = crc64(0, (unsigned char*)sdskey,sdslen(sdskey))&(TRACKING_TABLE_SIZE-1); - if (TrackingTable[hash] == NULL) return; - - raxIterator ri; - raxStart(&ri,TrackingTable[hash]); - raxSeek(&ri,"^",NULL,0); - while(raxNext(&ri)) { - uint64_t id; - memcpy(&id,ri.key,ri.key_len); - client *c = lookupClientByID(id); - if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; - sendTrackingMessage(c,hash); - } - raxStop(&ri); - - /* Free the tracking table: we'll create the radix tree and populate it - * again if more keys will be modified in this hash slot. */ - raxFree(TrackingTable[hash]); - TrackingTable[hash] = NULL; - TrackingTableUsedSlots--; + trackingInvalidateSlot(hash); } /* This function is called when one or all the Redis databases are flushed @@ -235,3 +241,17 @@ void trackingInvalidateKeysOnFlush(int dbid) { } } } + +/* Tracking forces Redis to remember information about which client may have + * keys about certian caching slots. In workloads where there are a lot of + * reads, but keys are hardly modified, the amount of information we have + * to remember server side could be a lot: for each 16 millions of caching + * slots we may end with a radix tree containing many entries. + * + * So Redis allows the user to configure a maximum fill rate for the + * invalidation table. This function makes sure that we don't go over the + * specified fill rate: if we are over, we can just evict informations about + * random caching slots, and send invalidation messages to clients like if + * the key was modified. */ +void trackingLimitUsedSlots(void) { +} From ff0780e8e6a3101666a70e28952163eff648fa5c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 21 Jul 2019 17:41:03 +0300 Subject: [PATCH 028/320] Implement module api for aux data in rdb Other changes: * fix memory leak in error handling of rdb loading of type OBJ_MODULE --- runtest-moduleapi | 2 +- src/module.c | 41 ++++++ src/rdb.c | 87 ++++++++++-- src/rdb.h | 1 + src/redismodule.h | 11 +- src/server.h | 10 ++ tests/modules/Makefile | 6 +- tests/modules/testrdb.c | 229 +++++++++++++++++++++++++++++++ tests/unit/moduleapi/testrdb.tcl | 62 +++++++++ 9 files changed, 431 insertions(+), 18 deletions(-) create mode 100644 tests/modules/testrdb.c create mode 100644 tests/unit/moduleapi/testrdb.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..8e1c0cb23 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/testrdb "${@}" diff --git a/src/module.c b/src/module.c index f4f753c00..812a54e04 100644 --- a/src/module.c +++ b/src/module.c @@ -29,6 +29,7 @@ #include "server.h" #include "cluster.h" +#include "rdb.h" #include #define REDISMODULE_CORE 1 @@ -3078,6 +3079,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; + struct { + moduleTypeAuxLoadFunc aux_load; + moduleTypeAuxSaveFunc aux_save; + int aux_save_triggers; + } v2; } *tms = (struct typemethods*) typemethods_ptr; moduleType *mt = zcalloc(sizeof(*mt)); @@ -3089,6 +3095,11 @@ moduleType *RM_CreateDataType(RedisModuleCtx *ctx, const char *name, int encver, mt->mem_usage = tms->mem_usage; mt->digest = tms->digest; mt->free = tms->free; + if (tms->version >= 2) { + mt->aux_load = tms->v2.aux_load; + mt->aux_save = tms->v2.aux_save; + mt->aux_save_triggers = tms->v2.aux_save_triggers; + } memcpy(mt->name,name,sizeof(mt->name)); listAddNodeTail(ctx->module->types,mt); return mt; @@ -3355,6 +3366,36 @@ loaderr: return 0; /* Never reached. */ } +/* Iterate over modules, and trigger rdb aux saving for the ones modules types + * who asked for it. */ +ssize_t rdbSaveModulesAux(rio *rdb, int when) { + size_t total_written = 0; + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + listIter li; + listNode *ln; + + listRewind(module->types,&li); + while((ln = listNext(&li))) { + moduleType *mt = ln->value; + if (!mt->aux_save || !(mt->aux_save_triggers & when)) + continue; + ssize_t ret = rdbSaveSingleModuleAux(rdb, when, mt); + if (ret==-1) { + dictReleaseIterator(di); + return -1; + } + total_written += ret; + } + } + + dictReleaseIterator(di); + return total_written; +} + /* -------------------------------------------------------------------------- * Key digest API (DEBUG DIGEST interface for modules types) * -------------------------------------------------------------------------- */ diff --git a/src/rdb.c b/src/rdb.c index 2118a00f4..4e00fad67 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -971,7 +971,6 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { RedisModuleIO io; moduleValue *mv = o->ptr; moduleType *mt = mv->type; - moduleInitIOContext(io,mt,rdb,key); /* Write the "module" identifier as prefix, so that we'll be able * to call the right module during loading. */ @@ -980,10 +979,13 @@ ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { io.bytes += retval; /* Then write the module-specific representation + EOF marker. */ + moduleInitIOContext(io,mt,rdb,key); mt->rdb_save(&io,mv->value); retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF); - if (retval == -1) return -1; - io.bytes += retval; + if (retval == -1) + io.error = 1; + else + io.bytes += retval; if (io.ctx) { moduleFreeContext(io.ctx); @@ -1101,6 +1103,40 @@ int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) { return 1; } +ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) { + /* Save a module-specific aux value. */ + RedisModuleIO io; + int retval = rdbSaveType(rdb, RDB_OPCODE_MODULE_AUX); + + /* Write the "module" identifier as prefix, so that we'll be able + * to call the right module during loading. */ + retval = rdbSaveLen(rdb,mt->id); + if (retval == -1) return -1; + io.bytes += retval; + + /* write the 'when' so that we can provide it on loading */ + retval = rdbSaveLen(rdb,when); + if (retval == -1) return -1; + io.bytes += retval; + + /* Then write the module-specific representation + EOF marker. */ + moduleInitIOContext(io,mt,rdb,NULL); + mt->aux_save(&io,when); + retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_EOF); + if (retval == -1) + io.error = 1; + else + io.bytes += retval; + + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + if (io.error) + return -1; + return io.bytes; +} + /* Produces a dump of the database in RDB format sending it to the specified * Redis I/O channel. On success C_OK is returned, otherwise C_ERR * is returned and part of the output, or all the output, can be @@ -1122,6 +1158,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION); if (rdbWriteRaw(rdb,magic,9) == -1) goto werr; if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr; + if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; @@ -1183,6 +1220,8 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { di = NULL; /* So that we don't release it again on error. */ } + if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_AFTER_RDB) == -1) goto werr; + /* EOF opcode */ if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr; @@ -2089,15 +2128,11 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { decrRefCount(auxval); continue; /* Read type again. */ } else if (type == RDB_OPCODE_MODULE_AUX) { - /* This is just for compatibility with the future: we have plans - * to add the ability for modules to store anything in the RDB - * file, like data that is not related to the Redis key space. - * Such data will potentially be stored both before and after the - * RDB keys-values section. For this reason since RDB version 9, - * we have the ability to read a MODULE_AUX opcode followed by an - * identifier of the module, and a serialized value in "MODULE V2" - * format. */ + /* Load module data that is not related to the Redis key space. + * Such data can be potentially be stored both before and after the + * RDB keys-values section. */ uint64_t moduleid = rdbLoadLen(rdb,NULL); + int when = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) goto eoferr; moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; @@ -2108,10 +2143,32 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load: no matching module '%s'", name); exit(1); } else if (!rdbCheckMode && mt != NULL) { - /* This version of Redis actually does not know what to do - * with modules AUX data... */ - serverLog(LL_WARNING,"The RDB file contains AUX module data I can't load for the module '%s'. Probably you want to use a newer version of Redis which implements aux data callbacks", name); - exit(1); + if (!mt->aux_load) { + /* Module doesn't support AUX. */ + serverLog(LL_WARNING,"The RDB file contains module AUX data, but the module '%s' doesn't seem to support it.", name); + exit(1); + } + + RedisModuleIO io; + moduleInitIOContext(io,mt,rdb,NULL); + io.ver = 2; + /* Call the rdb_load method of the module providing the 10 bit + * encoding version in the lower 10 bits of the module ID. */ + if (mt->aux_load(&io,moduleid&1023, when) || io.error) { + moduleTypeNameByID(name,moduleid); + serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); + exit(1); + } + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + uint64_t eof = rdbLoadLen(rdb,NULL); + if (eof != RDB_MODULE_OPCODE_EOF) { + serverLog(LL_WARNING,"The RDB file contains module AUX data for the module '%s' that is not terminated by the proper module value EOF marker", name); + exit(1); + } + continue; } else { /* RDB check mode. */ robj *aux = rdbLoadCheckModuleValue(rdb,name); diff --git a/src/rdb.h b/src/rdb.h index 0acddf9ab..40a50f7ba 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -145,6 +145,7 @@ size_t rdbSavedObjectLen(robj *o); robj *rdbLoadObject(int type, rio *rdb, robj *key); void backgroundSaveDoneHandler(int exitcode, int bysignal); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); +ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt); robj *rdbLoadStringObject(rio *rdb); ssize_t rdbSaveStringObject(rio *rdb, robj *obj); ssize_t rdbSaveRawString(rio *rdb, unsigned char *s, size_t len); diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..21227fe3a 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -129,6 +129,10 @@ #define REDISMODULE_NOT_USED(V) ((void) V) +/* Bit flags for aux_save_triggers and the aux_load and aux_save callbacks */ +#define REDISMODULE_AUX_BEFORE_RDB (1<<0) +#define REDISMODULE_AUX_AFTER_RDB (1<<1) + /* This type represents a timer handle, and is returned when a timer is * registered and used in order to invalidate a timer. It's just a 64 bit * number, because this is how each timer is represented inside the radix tree @@ -166,6 +170,8 @@ typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlocke typedef int (*RedisModuleNotificationFunc)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); typedef void *(*RedisModuleTypeLoadFunc)(RedisModuleIO *rdb, int encver); typedef void (*RedisModuleTypeSaveFunc)(RedisModuleIO *rdb, void *value); +typedef int (*RedisModuleTypeAuxLoadFunc)(RedisModuleIO *rdb, int encver, int when); +typedef void (*RedisModuleTypeAuxSaveFunc)(RedisModuleIO *rdb, int when); typedef void (*RedisModuleTypeRewriteFunc)(RedisModuleIO *aof, RedisModuleString *key, void *value); typedef size_t (*RedisModuleTypeMemUsageFunc)(const void *value); typedef void (*RedisModuleTypeDigestFunc)(RedisModuleDigest *digest, void *value); @@ -174,7 +180,7 @@ typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const cha typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); -#define REDISMODULE_TYPE_METHOD_VERSION 1 +#define REDISMODULE_TYPE_METHOD_VERSION 2 typedef struct RedisModuleTypeMethods { uint64_t version; RedisModuleTypeLoadFunc rdb_load; @@ -183,6 +189,9 @@ typedef struct RedisModuleTypeMethods { RedisModuleTypeMemUsageFunc mem_usage; RedisModuleTypeDigestFunc digest; RedisModuleTypeFreeFunc free; + RedisModuleTypeAuxLoadFunc aux_load; + RedisModuleTypeAuxSaveFunc aux_save; + int aux_save_triggers; } RedisModuleTypeMethods; #define REDISMODULE_GET_API(name) \ diff --git a/src/server.h b/src/server.h index b200a6696..33522dd69 100644 --- a/src/server.h +++ b/src/server.h @@ -536,6 +536,10 @@ typedef long long mstime_t; /* millisecond time type. */ #define REDISMODULE_TYPE_ENCVER(id) (id & REDISMODULE_TYPE_ENCVER_MASK) #define REDISMODULE_TYPE_SIGN(id) ((id & ~((uint64_t)REDISMODULE_TYPE_ENCVER_MASK)) >>REDISMODULE_TYPE_ENCVER_BITS) +/* Bit flags for moduleTypeAuxSaveFunc */ +#define REDISMODULE_AUX_BEFORE_RDB (1<<0) +#define REDISMODULE_AUX_AFTER_RDB (1<<1) + struct RedisModule; struct RedisModuleIO; struct RedisModuleDigest; @@ -548,6 +552,8 @@ struct redisObject; * is deleted. */ typedef void *(*moduleTypeLoadFunc)(struct RedisModuleIO *io, int encver); typedef void (*moduleTypeSaveFunc)(struct RedisModuleIO *io, void *value); +typedef int (*moduleTypeAuxLoadFunc)(struct RedisModuleIO *rdb, int encver, int when); +typedef void (*moduleTypeAuxSaveFunc)(struct RedisModuleIO *rdb, int when); typedef void (*moduleTypeRewriteFunc)(struct RedisModuleIO *io, struct redisObject *key, void *value); typedef void (*moduleTypeDigestFunc)(struct RedisModuleDigest *digest, void *value); typedef size_t (*moduleTypeMemUsageFunc)(const void *value); @@ -564,6 +570,9 @@ typedef struct RedisModuleType { moduleTypeMemUsageFunc mem_usage; moduleTypeDigestFunc digest; moduleTypeFreeFunc free; + moduleTypeAuxLoadFunc aux_load; + moduleTypeAuxSaveFunc aux_save; + int aux_save_triggers; char name[10]; /* 9 bytes name + null term. Charset: A-Z a-z 0-9 _- */ } moduleType; @@ -1528,6 +1537,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +ssize_t rdbSaveModulesAux(rio *rdb, int when); /* Utils */ long long ustime(void); diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20afa..6e4574747 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,16 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so testrdb.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +testrdb.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +testrdb.so: testrdb.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c new file mode 100644 index 000000000..415497a2f --- /dev/null +++ b/tests/modules/testrdb.c @@ -0,0 +1,229 @@ +#include "redismodule.h" + +#include +#include + +/* Module configuration, save aux or not? */ +long long conf_aux_count = 0; + +/* Registered type */ +RedisModuleType *testrdb_type = NULL; + +/* Global values to store and persist to aux */ +RedisModuleString *before_str = NULL; +RedisModuleString *after_str = NULL; + +void *testrdb_type_load(RedisModuleIO *rdb, int encver) { + int count = RedisModule_LoadSigned(rdb); + assert(count==1); + assert(encver==1); + RedisModuleString *str = RedisModule_LoadString(rdb); + return str; +} + +void testrdb_type_save(RedisModuleIO *rdb, void *value) { + RedisModuleString *str = (RedisModuleString*)value; + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, str); +} + +void testrdb_aux_save(RedisModuleIO *rdb, int when) { + if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB); + if (conf_aux_count==0) assert(0); + if (when == REDISMODULE_AUX_BEFORE_RDB) { + if (before_str) { + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, before_str); + } else { + RedisModule_SaveSigned(rdb, 0); + } + } else { + if (after_str) { + RedisModule_SaveSigned(rdb, 1); + RedisModule_SaveString(rdb, after_str); + } else { + RedisModule_SaveSigned(rdb, 0); + } + } +} + +int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { + assert(encver == 1); + if (conf_aux_count==1) assert(when == REDISMODULE_AUX_AFTER_RDB); + if (conf_aux_count==0) assert(0); + RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb); + if (when == REDISMODULE_AUX_BEFORE_RDB) { + if (before_str) + RedisModule_FreeString(ctx, before_str); + before_str = NULL; + int count = RedisModule_LoadSigned(rdb); + if (count) + before_str = RedisModule_LoadString(rdb); + } else { + if (after_str) + RedisModule_FreeString(ctx, after_str); + after_str = NULL; + int count = RedisModule_LoadSigned(rdb); + if (count) + after_str = RedisModule_LoadString(rdb); + } + return REDISMODULE_OK; +} + +void testrdb_type_free(void *value) { + RedisModule_FreeString(NULL, (RedisModuleString*)value); +} + +int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + if (before_str) + RedisModule_FreeString(ctx, before_str); + before_str = argv[1]; + RedisModule_RetainString(ctx, argv[1]); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + if (argc != 1){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + if (before_str) + RedisModule_ReplyWithString(ctx, before_str); + else + RedisModule_ReplyWithStringBuffer(ctx, "", 0); + return REDISMODULE_OK; +} + +int testrdb_set_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + if (after_str) + RedisModule_FreeString(ctx, after_str); + after_str = argv[1]; + RedisModule_RetainString(ctx, argv[1]); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_after(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + if (argc != 1){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + if (after_str) + RedisModule_ReplyWithString(ctx, after_str); + else + RedisModule_ReplyWithStringBuffer(ctx, "", 0); + return REDISMODULE_OK; +} + +int testrdb_set_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 3){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + RedisModuleString *str = RedisModule_ModuleTypeGetValue(key); + if (str) + RedisModule_FreeString(ctx, str); + RedisModule_ModuleTypeSetValue(key, testrdb_type, argv[2]); + RedisModule_RetainString(ctx, argv[2]); + RedisModule_CloseKey(key); + RedisModule_ReplyWithLongLong(ctx, 1); + return REDISMODULE_OK; +} + +int testrdb_get_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + RedisModuleString *str = RedisModule_ModuleTypeGetValue(key); + RedisModule_CloseKey(key); + RedisModule_ReplyWithString(ctx, str); + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (argc > 0) + RedisModule_StringToLongLong(argv[0], &conf_aux_count); + + if (conf_aux_count==0) { + RedisModuleTypeMethods datatype_methods = { + .version = 1, + .rdb_load = testrdb_type_load, + .rdb_save = testrdb_type_save, + .aof_rewrite = NULL, + .digest = NULL, + .free = testrdb_type_free, + }; + + testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods); + if (testrdb_type == NULL) + return REDISMODULE_ERR; + } else { + RedisModuleTypeMethods datatype_methods = { + .version = REDISMODULE_TYPE_METHOD_VERSION, + .rdb_load = testrdb_type_load, + .rdb_save = testrdb_type_save, + .aof_rewrite = NULL, + .digest = NULL, + .free = testrdb_type_free, + .aux_load = testrdb_aux_load, + .aux_save = testrdb_aux_save, + .aux_save_triggers = (conf_aux_count == 1 ? + REDISMODULE_AUX_AFTER_RDB : + REDISMODULE_AUX_BEFORE_RDB | REDISMODULE_AUX_AFTER_RDB) + }; + + testrdb_type = RedisModule_CreateDataType(ctx, "test__rdb", 1, &datatype_methods); + if (testrdb_type == NULL) + return REDISMODULE_ERR; + } + + if (RedisModule_CreateCommand(ctx,"testrdb.set.before", testrdb_set_before,"deny-oom",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.before", testrdb_get_before,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.set.after", testrdb_set_after,"deny-oom",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.after", testrdb_get_after,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.set.key", testrdb_set_key,"deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"testrdb.get.key", testrdb_get_key,"",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl new file mode 100644 index 000000000..22201a08e --- /dev/null +++ b/tests/unit/moduleapi/testrdb.tcl @@ -0,0 +1,62 @@ +set testmodule [file normalize tests/modules/testrdb.so] + +proc restart_and_wait {} { + catch { + r debug restart + } + + # wait for the server to come back up + set retry 50 + while {$retry} { + if {[catch { r ping }]} { + after 100 + } else { + break + } + incr retry -1 + } +} + +tags "modules" { + start_server [list overrides [list loadmodule "$testmodule"]] { + test {modules are able to persist types} { + r testrdb.set.key key1 value1 + assert_equal "value1" [r testrdb.get.key key1] + r debug reload + assert_equal "value1" [r testrdb.get.key key1] + } + + test {modules global are lost without aux} { + r testrdb.set.before global1 + assert_equal "global1" [r testrdb.get.before] + restart_and_wait + assert_equal "" [r testrdb.get.before] + } + } + + start_server [list overrides [list loadmodule "$testmodule 2"]] { + test {modules are able to persist globals before and after} { + r testrdb.set.before global1 + r testrdb.set.after global2 + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + restart_and_wait + assert_equal "global1" [r testrdb.get.before] + assert_equal "global2" [r testrdb.get.after] + } + + } + + start_server [list overrides [list loadmodule "$testmodule 1"]] { + test {modules are able to persist globals just after} { + r testrdb.set.after global2 + assert_equal "global2" [r testrdb.get.after] + restart_and_wait + assert_equal "global2" [r testrdb.get.after] + } + } + + + # TODO: test short read handling + +} From dedbc9d4d24146aec715b22261331aaae26e891e Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Sat, 6 Jul 2019 18:32:58 -0700 Subject: [PATCH 029/320] Removed unecessary creation of Redis objects --- src/cluster.c | 13 +++---------- src/config.c | 6 ++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index c85e3791d..e22222700 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4251,12 +4251,7 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - robj *o; - sds ci = clusterGenNodesDescription(0); - - o = createObject(OBJ_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); + addReplyBulkSds(c,clusterGenNodesDescription(0)); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4832,7 +4827,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) { * DUMP is actually not used by Redis Cluster but it is the obvious * complement of RESTORE and can be useful for different applications. */ void dumpCommand(client *c) { - robj *o, *dumpobj; + robj *o; rio payload; /* Check if the key is here. */ @@ -4845,9 +4840,7 @@ void dumpCommand(client *c) { createDumpPayload(&payload,o,c->argv[1]); /* Transfer to the client */ - dumpobj = createObject(OBJ_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); + addReplyBulkSds(c,payload.io.buffer.ptr); return; } diff --git a/src/config.c b/src/config.c index 7f0e9af89..ddf31d168 100644 --- a/src/config.c +++ b/src/config.c @@ -1602,12 +1602,10 @@ void configGetCommand(client *c) { matches++; } if (stringmatch(pattern,"notify-keyspace-events",1)) { - robj *flagsobj = createObject(OBJ_STRING, - keyspaceEventsFlagsToString(server.notify_keyspace_events)); + sds flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); addReplyBulkCString(c,"notify-keyspace-events"); - addReplyBulk(c,flagsobj); - decrRefCount(flagsobj); + addReplyBulkSds(c,flags); matches++; } if (stringmatch(pattern,"bind",1)) { From 4520e878002d08339b389fa33794999ee53061fb Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 19 Jul 2019 10:22:40 -0700 Subject: [PATCH 030/320] Hide HELLO and AUTH from slowlog and monitor --- src/server.c | 13 +++++++++---- src/server.h | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/src/server.c b/src/server.c index 2643d7266..0b79f6a62 100644 --- a/src/server.c +++ b/src/server.c @@ -146,6 +146,8 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */ * in this condition but just a few. * * no-monitor: Do not automatically propagate the command on MONITOR. + * + * no-slowlog: Do not automatically propagate the command to the slowlog. * * cluster-asking: Perform an implicit ASKING for this command, so the * command will be accepted in cluster mode if the slot is marked @@ -627,7 +629,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"auth",authCommand,-2, - "no-script ok-loading ok-stale fast @connection", + "no-script ok-loading ok-stale fast no-monitor no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* We don't allow PING during loading since in Redis PING is used as @@ -670,7 +672,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"exec",execCommand,1, - "no-script no-monitor @transaction", + "no-script no-monitor no-slowlog @transaction", 0,NULL,0,0,0,0,0,0}, {"discard",discardCommand,1, @@ -822,7 +824,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"hello",helloCommand,-2, - "no-script fast @connection", + "no-script fast no-monitor no-slowlog @connection", 0,NULL,0,0,0,0,0,0}, /* EVAL can modify the dataset, however it is not flagged as a write @@ -2902,6 +2904,8 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags) { c->flags |= CMD_STALE; } else if (!strcasecmp(flag,"no-monitor")) { c->flags |= CMD_SKIP_MONITOR; + } else if (!strcasecmp(flag,"no-slowlog")) { + c->flags |= CMD_SKIP_SLOWLOG; } else if (!strcasecmp(flag,"cluster-asking")) { c->flags |= CMD_ASKING; } else if (!strcasecmp(flag,"fast")) { @@ -3186,7 +3190,7 @@ void call(client *c, int flags) { /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ - if (flags & CMD_CALL_SLOWLOG && c->cmd->proc != execCommand) { + if (flags & CMD_CALL_SLOWLOG && !(c->flags & CMD_SKIP_SLOWLOG)) { char *latency_event = (c->cmd->flags & CMD_FAST) ? "fast-command" : "command"; latencyAddSampleIfNeeded(latency_event,duration/1000); @@ -3677,6 +3681,7 @@ void addReplyCommand(client *c, struct redisCommand *cmd) { flagcount += addReplyCommandFlag(c,cmd,CMD_LOADING, "loading"); flagcount += addReplyCommandFlag(c,cmd,CMD_STALE, "stale"); flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_MONITOR, "skip_monitor"); + flagcount += addReplyCommandFlag(c,cmd,CMD_SKIP_SLOWLOG, "skip_slowlog"); flagcount += addReplyCommandFlag(c,cmd,CMD_ASKING, "asking"); flagcount += addReplyCommandFlag(c,cmd,CMD_FAST, "fast"); if ((cmd->getkeys_proc && !(cmd->flags & CMD_MODULE)) || diff --git a/src/server.h b/src/server.h index 3987ab5fc..ecb54a3cf 100644 --- a/src/server.h +++ b/src/server.h @@ -218,35 +218,36 @@ typedef long long mstime_t; /* millisecond time type. */ #define CMD_LOADING (1ULL<<9) /* "ok-loading" flag */ #define CMD_STALE (1ULL<<10) /* "ok-stale" flag */ #define CMD_SKIP_MONITOR (1ULL<<11) /* "no-monitor" flag */ -#define CMD_ASKING (1ULL<<12) /* "cluster-asking" flag */ -#define CMD_FAST (1ULL<<13) /* "fast" flag */ +#define CMD_SKIP_SLOWLOG (1ULL<<12) /* "no-slowlog" flag */ +#define CMD_ASKING (1ULL<<13) /* "cluster-asking" flag */ +#define CMD_FAST (1ULL<<14) /* "fast" flag */ /* Command flags used by the module system. */ -#define CMD_MODULE_GETKEYS (1ULL<<14) /* Use the modules getkeys interface. */ -#define CMD_MODULE_NO_CLUSTER (1ULL<<15) /* Deny on Redis Cluster. */ +#define CMD_MODULE_GETKEYS (1ULL<<15) /* Use the modules getkeys interface. */ +#define CMD_MODULE_NO_CLUSTER (1ULL<<16) /* Deny on Redis Cluster. */ /* Command flags that describe ACLs categories. */ -#define CMD_CATEGORY_KEYSPACE (1ULL<<16) -#define CMD_CATEGORY_READ (1ULL<<17) -#define CMD_CATEGORY_WRITE (1ULL<<18) -#define CMD_CATEGORY_SET (1ULL<<19) -#define CMD_CATEGORY_SORTEDSET (1ULL<<20) -#define CMD_CATEGORY_LIST (1ULL<<21) -#define CMD_CATEGORY_HASH (1ULL<<22) -#define CMD_CATEGORY_STRING (1ULL<<23) -#define CMD_CATEGORY_BITMAP (1ULL<<24) -#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<25) -#define CMD_CATEGORY_GEO (1ULL<<26) -#define CMD_CATEGORY_STREAM (1ULL<<27) -#define CMD_CATEGORY_PUBSUB (1ULL<<28) -#define CMD_CATEGORY_ADMIN (1ULL<<29) -#define CMD_CATEGORY_FAST (1ULL<<30) -#define CMD_CATEGORY_SLOW (1ULL<<31) -#define CMD_CATEGORY_BLOCKING (1ULL<<32) -#define CMD_CATEGORY_DANGEROUS (1ULL<<33) -#define CMD_CATEGORY_CONNECTION (1ULL<<34) -#define CMD_CATEGORY_TRANSACTION (1ULL<<35) -#define CMD_CATEGORY_SCRIPTING (1ULL<<36) +#define CMD_CATEGORY_KEYSPACE (1ULL<<17) +#define CMD_CATEGORY_READ (1ULL<<18) +#define CMD_CATEGORY_WRITE (1ULL<<19) +#define CMD_CATEGORY_SET (1ULL<<20) +#define CMD_CATEGORY_SORTEDSET (1ULL<<21) +#define CMD_CATEGORY_LIST (1ULL<<22) +#define CMD_CATEGORY_HASH (1ULL<<23) +#define CMD_CATEGORY_STRING (1ULL<<24) +#define CMD_CATEGORY_BITMAP (1ULL<<25) +#define CMD_CATEGORY_HYPERLOGLOG (1ULL<<26) +#define CMD_CATEGORY_GEO (1ULL<<27) +#define CMD_CATEGORY_STREAM (1ULL<<28) +#define CMD_CATEGORY_PUBSUB (1ULL<<29) +#define CMD_CATEGORY_ADMIN (1ULL<<30) +#define CMD_CATEGORY_FAST (1ULL<<31) +#define CMD_CATEGORY_SLOW (1ULL<<32) +#define CMD_CATEGORY_BLOCKING (1ULL<<33) +#define CMD_CATEGORY_DANGEROUS (1ULL<<34) +#define CMD_CATEGORY_CONNECTION (1ULL<<35) +#define CMD_CATEGORY_TRANSACTION (1ULL<<36) +#define CMD_CATEGORY_SCRIPTING (1ULL<<37) /* AOF states */ #define AOF_OFF 0 /* AOF is off */ From 408ca31d16e839a47807269e3886e962e12947b8 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 23 Jul 2019 15:25:00 +0800 Subject: [PATCH 031/320] Client side caching: do not reclaim tracking table if it's empty --- src/tracking.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tracking.c b/src/tracking.c index 1189e82d5..badec69a7 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -206,8 +206,6 @@ void trackingInvalidateKey(robj *keyobj) { * flush the table: it will slowly get garbage collected as more keys * are modified in the used caching slots. */ void trackingInvalidateKeysOnFlush(int dbid) { - UNUSED(dbid); - if (server.tracking_clients) { listNode *ln; listIter li; @@ -222,12 +220,11 @@ void trackingInvalidateKeysOnFlush(int dbid) { /* In case of FLUSHALL, reclaim all the memory used by tracking. */ if (dbid == -1 && TrackingTable) { - for (int j = 0; j < TRACKING_TABLE_SIZE; j++) { + for (int j = 0; j < TRACKING_TABLE_SIZE && TrackingTableUsedSlots > 0; j++) { if (TrackingTable[j] != NULL) { raxFree(TrackingTable[j]); TrackingTable[j] = NULL; TrackingTableUsedSlots--; - if (TrackingTableUsedSlots == 0) break; } } From 79a73bb95a0c59bce292f8aba8e2b59a0851c3f3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Jul 2019 10:57:22 +0200 Subject: [PATCH 032/320] Client side caching: implement full slot limit function. --- src/server.c | 3 +++ src/server.h | 2 ++ src/tracking.c | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/server.c b/src/server.c index 5e18b5465..1f502e4ff 100644 --- a/src/server.c +++ b/src/server.c @@ -2403,6 +2403,9 @@ void initServerConfig(void) { /* Latency monitor */ server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; + /* Tracking. */ + server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_MAX_FILL; + /* Debugging */ server.assert_failed = ""; server.assert_file = ""; diff --git a/src/server.h b/src/server.h index fdab53830..172b1cf64 100644 --- a/src/server.h +++ b/src/server.h @@ -171,6 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ +#define CONFIG_DEFAULT_TRACKING_MAX_FILL 10 /* 10% tracking table max fill. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ @@ -1316,6 +1317,7 @@ struct redisServer { list *ready_keys; /* List of readyList structures for BLPOP & co */ /* Client side caching. */ unsigned int tracking_clients; /* # of clients with tracking enabled.*/ + int tracking_table_max_fill; /* Max fill percentage. */ /* Sort parameters - qsort_r() is only available under BSD so we * have to take this state global, in order to pass it to sortCompare() */ int sort_desc; diff --git a/src/tracking.c b/src/tracking.c index badec69a7..1a5fbe47c 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -251,4 +251,40 @@ void trackingInvalidateKeysOnFlush(int dbid) { * random caching slots, and send invalidation messages to clients like if * the key was modified. */ void trackingLimitUsedSlots(void) { + static unsigned int timeout_counter = 0; + + if (server.tracking_table_max_fill == 0) return; /* No limits set. */ + unsigned int max_slots = + (TRACKING_TABLE_SIZE/100) * server.tracking_table_max_fill; + if (TrackingTableUsedSlots <= max_slots) { + timeout_counter = 0; + return; /* Limit not reached. */ + } + + /* We have to invalidate a few slots to reach the limit again. The effort + * we do here is proportional to the number of times we entered this + * function and found that we are still over the limit. */ + int effort = 100 * (timeout_counter+1); + + /* Let's start at a random position, and perform linear probing, in order + * to improve cache locality. However once we are able to find an used + * slot, jump again randomly, in order to avoid creating big holes in the + * table (that will make this funciton use more resourced later). */ + while(effort > 0) { + unsigned int idx = rand() % TRACKING_TABLE_SIZE; + do { + effort--; + idx = (idx+1) % TRACKING_TABLE_SIZE; + if (TrackingTable[idx] != NULL) { + trackingInvalidateSlot(idx); + if (TrackingTableUsedSlots <= max_slots) { + timeout_counter = 0; + return; /* Return ASAP: we are again under the limit. */ + } else { + break; /* Jump to next random position. */ + } + } + } while(effort > 0); + } + timeout_counter++; } From 1d54931685f8e9137d04112b45f51c25c0cd30b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Jul 2019 11:02:14 +0200 Subject: [PATCH 033/320] Client side caching: show tracking slots usage in INFO. --- src/server.c | 6 ++++-- src/server.h | 1 + src/tracking.c | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 1f502e4ff..0acf8545c 100644 --- a/src/server.c +++ b/src/server.c @@ -4147,7 +4147,8 @@ sds genRedisInfoString(char *section) { "active_defrag_hits:%lld\r\n" "active_defrag_misses:%lld\r\n" "active_defrag_key_hits:%lld\r\n" - "active_defrag_key_misses:%lld\r\n", + "active_defrag_key_misses:%lld\r\n" + "tracking_used_slots:%lld\r\n", server.stat_numconnections, server.stat_numcommands, getInstantaneousMetric(STATS_METRIC_COMMAND), @@ -4173,7 +4174,8 @@ sds genRedisInfoString(char *section) { server.stat_active_defrag_hits, server.stat_active_defrag_misses, server.stat_active_defrag_key_hits, - server.stat_active_defrag_key_misses); + server.stat_active_defrag_key_misses, + trackingGetUsedSlots()); } /* Replication */ diff --git a/src/server.h b/src/server.h index 172b1cf64..e363f03f4 100644 --- a/src/server.h +++ b/src/server.h @@ -1642,6 +1642,7 @@ void trackingRememberKeys(client *c); void trackingInvalidateKey(robj *keyobj); void trackingInvalidateKeysOnFlush(int dbid); void trackingLimitUsedSlots(void); +unsigned long long trackingGetUsedSlots(void); /* List data type */ void listTypeTryConversion(robj *subject, robj *value); diff --git a/src/tracking.c b/src/tracking.c index 1a5fbe47c..f7f0fc755 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -288,3 +288,9 @@ void trackingLimitUsedSlots(void) { } timeout_counter++; } + +/* This is just used in order to access the amount of used slots in the + * tracking table. */ +unsigned long long trackingGetUsedSlots(void) { + return TrackingTableUsedSlots; +} From 61fe76a7b9c3b05f3062d479601ce7bb2ac3577d Mon Sep 17 00:00:00 2001 From: wubostc <913721086@qq.com> Date: Wed, 24 Jul 2019 16:22:26 +0800 Subject: [PATCH 034/320] Reduce the calling stack --- src/t_zset.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index fb7078abd..2680e76a9 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1357,9 +1357,8 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { /* Optimize: check if the element is too large or the list * becomes too long *before* executing zzlInsert. */ zobj->ptr = zzlInsert(zobj->ptr,ele,score); - if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries) - zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); - if (sdslen(ele) > server.zset_max_ziplist_value) + if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries || + sdslen(ele) > server.zset_max_ziplist_value) zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); if (newscore) *newscore = score; *flags |= ZADD_ADDED; From 1ead7a115817e60d61a2ffe4c774c46273f2787d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:33:53 +0200 Subject: [PATCH 035/320] Remove experimental warning from diskless replication. --- redis.conf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/redis.conf b/redis.conf index 74b6c018f..a3c98341f 100644 --- a/redis.conf +++ b/redis.conf @@ -336,10 +336,6 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- -# # New replicas and reconnecting replicas that are not able to continue the replication # process just receiving differences, need to do what is called a "full # synchronization". An RDB file is transmitted from the master to the replicas. From 83b8fbf58020ac26045dbc32c7332e960960b33a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:35:01 +0200 Subject: [PATCH 036/320] Client side caching: config option for table fill rate. --- src/config.c | 18 ++++++++++++++++-- src/server.c | 2 +- src/server.h | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fde00ddf5..7ffc94b35 100644 --- a/src/config.c +++ b/src/config.c @@ -686,6 +686,17 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { server.slowlog_max_len = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"tracking-table-max-fill") && + argc == 2) + { + server.tracking_table_max_fill = strtoll(argv[1],NULL,10); + if (server.tracking_table_max_fill > 100 || + server.tracking_table_max_fill < 0) + { + err = "The tracking table fill percentage must be an " + "integer between 0 and 100"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -1133,6 +1144,8 @@ void configSetCommand(client *c) { "slowlog-max-len",ll,0,LONG_MAX) { /* Cast to unsigned. */ server.slowlog_max_len = (unsigned long)ll; + } config_set_numerical_field( + "tracking-table-max-fill",server.tracking_table_max_fill,0,100) { } config_set_numerical_field( "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ } config_set_numerical_field( @@ -1338,8 +1351,8 @@ void configGetCommand(client *c) { server.slowlog_log_slower_than); config_get_numerical_field("latency-monitor-threshold", server.latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", - server.slowlog_max_len); + config_get_numerical_field("slowlog-max-len", server.slowlog_max_len); + config_get_numerical_field("tracking-table-max-fill", server.tracking_table_max_fill); config_get_numerical_field("port",server.port); config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); @@ -2167,6 +2180,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); + rewriteConfigNumericalOption(state,"tracking-table-max-fill",server.tracking_table_max_fill,CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); diff --git a/src/server.c b/src/server.c index 0acf8545c..eb5cef386 100644 --- a/src/server.c +++ b/src/server.c @@ -2404,7 +2404,7 @@ void initServerConfig(void) { server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; /* Tracking. */ - server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_MAX_FILL; + server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL; /* Debugging */ server.assert_failed = ""; diff --git a/src/server.h b/src/server.h index e363f03f4..399a0bbb6 100644 --- a/src/server.h +++ b/src/server.h @@ -171,7 +171,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ -#define CONFIG_DEFAULT_TRACKING_MAX_FILL 10 /* 10% tracking table max fill. */ +#define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ From 27095db3f3b7473e788f144b6ce666a6bbd8f5e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:38:11 +0200 Subject: [PATCH 037/320] Example redis.conf: stay under 80 cols. --- redis.conf | 64 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/redis.conf b/redis.conf index a3c98341f..28a9889ea 100644 --- a/redis.conf +++ b/redis.conf @@ -336,9 +336,11 @@ replica-read-only yes # Replication SYNC strategy: disk or socket. # -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. +# New replicas and reconnecting replicas that are not able to continue the +# replication process just receiving differences, need to do what is called a +# "full synchronization". An RDB file is transmitted from the master to the +# replicas. +# # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB @@ -348,14 +350,14 @@ replica-read-only yes # RDB file to replica sockets, without touching the disk at all. # # With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. +# can be queued and served with the RDB file as soon as the current child +# producing the RDB file finishes its work. With diskless replication instead +# once the transfer starts, new replicas arriving will be queued and a new +# transfer will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. +# time (in seconds) before starting the transfer in the hope that multiple +# replicas will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication # works better. @@ -366,8 +368,8 @@ repl-diskless-sync no # to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the +# server waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. @@ -389,9 +391,9 @@ repl-diskless-sync-delay 5 # sufficient memory, if you don't have it, you risk an OOM kill. repl-diskless-load disabled -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. +# Replicas send PINGs to server in a predefined interval. It's possible to +# change this interval with the repl_ping_replica_period option. The default +# value is 10 seconds. # # repl-ping-replica-period 10 @@ -423,10 +425,10 @@ repl-diskless-load disabled repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. +# replica data when replicas are disconnected for some time, so that when a +# replica wants to reconnect again, often a full resync is not needed, but a +# partial resync is enough, just passing the portion of data the replica +# missed while disconnected. # # The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. @@ -448,13 +450,13 @@ repl-disable-tcp-nodelay no # # repl-backlog-ttl 3600 -# The replica priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. +# The replica priority is an integer number published by Redis in the INFO +# output. It is used by Redis Sentinel in order to select a replica to promote +# into a master if the master is no longer working correctly. # # A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. +# for instance if there are three replicas with priority 10, 100, 25 Sentinel +# will pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the replica as not able to perform the # role of master, so a replica with priority of 0 will never be selected by @@ -743,17 +745,17 @@ replica-priority 100 # DEL commands to the replica as keys evict in the master side. # # This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). +# what you want, however if your replica is writable, or you want the replica +# to have a different memory setting, and you are sure all the writes performed +# to the replica are idempotent, then you may change this default (but be sure +# to understand what you are doing). # # Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. +# be larger on the replica, or data structures may sometimes take more memory +# and so forth). So make sure you monitor your replicas and make sure they +# have enough memory to never hit a real out-of-memory condition before the +# master hits the configured maxmemory setting. # # replica-ignore-maxmemory yes From 63e1d1cb27bf4106cc4684e18800fb9f2aa5fb2f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2019 11:45:10 +0200 Subject: [PATCH 038/320] Client side caching: document tracking-table-max-fill in redis.conf. --- redis.conf | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/redis.conf b/redis.conf index 28a9889ea..c242609ec 100644 --- a/redis.conf +++ b/redis.conf @@ -516,6 +516,39 @@ replica-priority 100 # replica-announce-ip 5.5.5.5 # replica-announce-port 1234 +############################### KEYS TRACKING ################################# + +# Redis implements server assisted support for client side caching of values. +# This is implemented using an invalidation table that remembers, using +# 16 millions of slots, what clients may have certain subsets of keys. In turn +# this is used in order to send invalidation messages to clients. Please +# to understand more about the feature check this page: +# +# https://redis.io/topics/client-side-caching +# +# When tracking is enabled for a client, all the read only queries are assumed +# to be cached: this will force Redis to store information in the invalidation +# table. When keys are modified, such information is flushed away, and +# invalidation messages are sent to the clients. However if the workload is +# heavily dominated by reads, Redis could use more and more memory in order +# to track the keys fetched by many clients. +# +# For this reason it is possible to configure a maximum fill value for the +# invalidation table. By default it is set to 10%, and once this limit is +# reached, Redis will start to evict caching slots in the invalidation table +# even if keys are not modified, just to reclaim memory: this will in turn +# force the clients to invalidate the cached values. Basically the table +# maximum fill rate is a trade off between the memory you want to spend server +# side to track information about who cached what, and the ability of clients +# to retain cached objects in memory. +# +# If you set the value to 0, it means there are no limits, and all the 16 +# millions of caching slots can be used at the same time. In the "stats" +# INFO section, you can find information about the amount of caching slots +# used at every given moment. +# +# tracking-table-max-fill 10 + ################################## SECURITY ################################### # Warning: since Redis is pretty fast an outside user can try up to From b2fce8069556162759f44e961d0e6c36a6c5c410 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 24 Jul 2019 12:58:15 +0300 Subject: [PATCH 039/320] Extend modules API to allow modules report to redis INFO this implements #6012 --- runtest-moduleapi | 2 +- src/debug.c | 6 ++ src/module.c | 130 ++++++++++++++++++++++++++++++ src/redismodule.h | 16 ++++ src/server.c | 16 +++- src/server.h | 1 + tests/modules/Makefile | 7 +- tests/modules/infotest.c | 32 ++++++++ tests/unit/moduleapi/infotest.tcl | 53 ++++++++++++ 9 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 tests/modules/infotest.c create mode 100644 tests/unit/moduleapi/infotest.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 84cdb9bb8..9bf12677d 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/infotest "${@}" diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..daf00b14c 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1337,6 +1337,12 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Log dump of processor registers */ logRegisters(uc); + /* Log Modules INFO */ + serverLogRaw(LL_WARNING|LL_RAW, "\n------ MODULES INFO OUTPUT ------\n"); + infostring = modulesCollectInfo(sdsempty(), NULL, 1, 0); + serverLogRaw(LL_WARNING|LL_RAW, infostring); + sdsfree(infostring); + #if defined(HAVE_PROC_MAPS) /* Test memory */ serverLogRaw(LL_WARNING|LL_RAW, "\n------ FAST MEMORY TEST ------\n"); diff --git a/src/module.c b/src/module.c index f4f753c00..0e19c11e2 100644 --- a/src/module.c +++ b/src/module.c @@ -40,6 +40,16 @@ * pointers that have an API the module can call with them) * -------------------------------------------------------------------------- */ +typedef struct RedisModuleInfoCtx { + struct RedisModule *module; + sds requested_section; + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ +} RedisModuleInfoCtx; + +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); + /* This structure represents a module inside the system. */ struct RedisModule { void *handle; /* Module dlopen() handle. */ @@ -51,6 +61,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + RedisModuleInfoFunc info_cb; /* callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -4686,6 +4697,118 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k return res ? REDISMODULE_OK : REDISMODULE_ERR; } + + + +/* -------------------------------------------------------------------------- + * Modules Info fields + * -------------------------------------------------------------------------- */ + +/* Used to start a new section, before adding any fields. the section name will + * be prefixed by "_" and must only include A-Z,a-z,0-9. + * When return value is REDISMODULE_ERR, the section should and will be skipped. */ +int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { + sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + + /* proceed only if: + * 1) no section was requested (emit all) + * 2) the module name was requested (emit all) + * 3) this specific section was requested. */ + if (ctx->requested_section) { + if (strcasecmp(ctx->requested_section, full_name) && + strcasecmp(ctx->requested_section, ctx->module->name)) { + sdsfree(full_name); + ctx->in_section = 0; + return REDISMODULE_ERR; + } + } + if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); + ctx->info = sdscatprintf(ctx->info, "# %s\r\n", full_name); + ctx->in_section = 1; + sdsfree(full_name); + return REDISMODULE_OK; +} + +/* Used by RedisModuleInfoFunc to add info fields. + * Each field will be automatically prefixed by "_". + * Field names or values must not include \r\n of ":" */ +int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + (sds)value->ptr); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%s\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%.17g\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%lld\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_AddInfoFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { + if (!ctx->in_section) + return REDISMODULE_ERR; + ctx->info = sdscatprintf(ctx->info, + "%s_%s:%llu\r\n", + ctx->module->name, + field, + value); + return REDISMODULE_OK; +} + +int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { + ctx->module->info_cb = cb; + return REDISMODULE_OK; +} + +sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections) { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; + + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (!module->info_cb) + continue; + RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; + module->info_cb(&info_ctx, for_crash_report); + info = info_ctx.info; + sections = info_ctx.sections; + } + dictReleaseIterator(di); + return info; +} + /* -------------------------------------------------------------------------- * Modules utility APIs * -------------------------------------------------------------------------- */ @@ -5490,4 +5613,11 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgInsert); REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); + REGISTER_API(RegisterInfoFunc); + REGISTER_API(AddInfoSection); + REGISTER_API(AddInfoFieldString); + REGISTER_API(AddInfoFieldCString); + REGISTER_API(AddInfoFieldDouble); + REGISTER_API(AddInfoFieldLongLong); + REGISTER_API(AddInfoFieldULongLong); } diff --git a/src/redismodule.h b/src/redismodule.h index b9c73957b..1feb14a68 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -160,6 +160,7 @@ typedef struct RedisModuleDict RedisModuleDict; typedef struct RedisModuleDictIter RedisModuleDictIter; typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx; typedef struct RedisModuleCommandFilter RedisModuleCommandFilter; +typedef struct RedisModuleInfoCtx RedisModuleInfoCtx; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -173,6 +174,7 @@ typedef void (*RedisModuleTypeFreeFunc)(void *value); typedef void (*RedisModuleClusterMessageReceiver)(RedisModuleCtx *ctx, const char *sender_id, uint8_t type, const unsigned char *payload, uint32_t len); typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); +typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); #define REDISMODULE_TYPE_METHOD_VERSION 1 typedef struct RedisModuleTypeMethods { @@ -317,6 +319,13 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictNext)(RedisModuleCtx *ct RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ctx, RedisModuleDictIter *di, void **dataptr); int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); +int REDISMODULE_API_FUNC(RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb); +int REDISMODULE_API_FUNC(RedisModule_AddInfoSection)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); +int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -490,6 +499,13 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(DictPrev); REDISMODULE_GET_API(DictCompare); REDISMODULE_GET_API(DictCompareC); + REDISMODULE_GET_API(RegisterInfoFunc); + REDISMODULE_GET_API(AddInfoSection); + REDISMODULE_GET_API(AddInfoFieldString); + REDISMODULE_GET_API(AddInfoFieldCString); + REDISMODULE_GET_API(AddInfoFieldDouble); + REDISMODULE_GET_API(AddInfoFieldLongLong); + REDISMODULE_GET_API(AddInfoFieldULongLong); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/src/server.c b/src/server.c index 4337b8f01..c671e375d 100644 --- a/src/server.c +++ b/src/server.c @@ -3809,12 +3809,15 @@ sds genRedisInfoString(char *section) { time_t uptime = server.unixtime-server.stat_starttime; int j; struct rusage self_ru, c_ru; - int allsections = 0, defsections = 0; + int allsections = 0, defsections = 0, everything = 0, modules = 0; int sections = 0; if (section == NULL) section = "default"; allsections = strcasecmp(section,"all") == 0; defsections = strcasecmp(section,"default") == 0; + everything = strcasecmp(section,"everything") == 0; + modules = strcasecmp(section,"modules") == 0; + if (everything) allsections = 1; getrusage(RUSAGE_SELF, &self_ru); getrusage(RUSAGE_CHILDREN, &c_ru); @@ -4357,6 +4360,17 @@ sds genRedisInfoString(char *section) { } } } + + /* Get info from modules. + * if user asked for "everything" or "modules", or a specific section + * that's not found yet. */ + if (everything || modules || + (!allsections && !defsections && sections==0)) { + info = modulesCollectInfo(info, + everything || modules ? NULL: section, + 0, /* not a crash report */ + sections); + } return info; } diff --git a/src/server.h b/src/server.h index b200a6696..ddddd84f0 100644 --- a/src/server.h +++ b/src/server.h @@ -1528,6 +1528,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); /* Utils */ long long ustime(void); diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 014d20afa..66bf6de35 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,12 +13,17 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so +all: commandfilter.so infotest.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ commandfilter.xo: ../../src/redismodule.h +infotest.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +infotest.so: infotest.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c new file mode 100644 index 000000000..d53ea2126 --- /dev/null +++ b/tests/modules/infotest.c @@ -0,0 +1,32 @@ +#include "redismodule.h" + +#include + +void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { + RedisModule_AddInfoSection(ctx, "Spanish"); + RedisModule_AddInfoFieldCString(ctx, "uno", "one"); + RedisModule_AddInfoFieldLongLong(ctx, "dos", 2); + + RedisModule_AddInfoSection(ctx, "Italian"); + RedisModule_AddInfoFieldLongLong(ctx, "due", 2); + RedisModule_AddInfoFieldDouble(ctx, "tre", 3.3); + + if (for_crash_report) { + RedisModule_AddInfoSection(ctx, "Klingon"); + RedisModule_AddInfoFieldCString(ctx, "one", "wa’"); + RedisModule_AddInfoFieldCString(ctx, "two", "cha’"); + RedisModule_AddInfoFieldCString(ctx, "three", "wej"); + } + +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + if (RedisModule_Init(ctx,"infotest",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_RegisterInfoFunc(ctx, InfoFunc) == REDISMODULE_ERR) return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl new file mode 100644 index 000000000..143f9050a --- /dev/null +++ b/tests/unit/moduleapi/infotest.tcl @@ -0,0 +1,53 @@ +set testmodule [file normalize tests/modules/infotest.so] + +# Return value for INFO property +proc field {info property} { + if {[regexp "\r\n$property:(.*?)\r\n" $info _ value]} { + set _ $value + } +} + +start_server {tags {"modules"}} { + r module load $testmodule log-key 0 + + test {module info all} { + set info [r info all] + # info all does not contain modules + assert { ![string match "*Spanish*" $info] } + assert { [string match "*used_memory*" $info] } + } + + test {module info everything} { + set info [r info everything] + # info everything contains all default sections, but not ones for crash report + assert { [string match "*Spanish*" $info] } + assert { [string match "*Italian*" $info] } + assert { [string match "*used_memory*" $info] } + assert { ![string match "*Klingon*" $info] } + field $info infotest_dos + } {2} + + test {module info modules} { + set info [r info modules] + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + } + + test {module info one module} { + set info [r info INFOTEST] + # info all does not contain modules + assert { [string match "*Spanish*" $info] } + assert { ![string match "*used_memory*" $info] } + } + + test {module info one section} { + set info [r info INFOTEST_SPANISH] + assert { ![string match "*used_memory*" $info] } + assert { ![string match "*Italian*" $info] } + field $info infotest_uno + } {one} + + # TODO: test crash report. + +} From 4e547a0ac5ab577b1abe5935137f56a031c2e06f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 25 Jul 2019 19:17:58 +0200 Subject: [PATCH 040/320] Mark diskless loads as experimental in redis.conf. --- redis.conf | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/redis.conf b/redis.conf index c242609ec..50ba823ac 100644 --- a/redis.conf +++ b/redis.conf @@ -375,15 +375,25 @@ repl-diskless-sync no # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Replica can load the rdb it reads from the replication link directly from the -# socket, or store the rdb to a file and read that file after it was completely +# ----------------------------------------------------------------------------- +# WARNING: RDB diskless load is experimental. Since in this setup the replica +# does not immediately store an RDB on disk, it may cause data loss during +# failovers. RDB diskless load + Redis modules not handling I/O reads may also +# cause Redis to abort in case of I/O errors during the initial synchronization +# stage with the master. Use only if your do what you are doing. +# ----------------------------------------------------------------------------- +# +# Replica can load the RDB it reads from the replication link directly from the +# socket, or store the RDB to a file and read that file after it was completely # recived from the master. +# # In many cases the disk is slower than the network, and storing and loading -# the rdb file may increase replication time (and even increase the master's +# the RDB file may increase replication time (and even increase the master's # Copy on Write memory and salve buffers). -# However, parsing the rdb file directly from the socket may mean that we have -# to flush the contents of the current database before the full rdb was received. -# for this reason we have the following options: +# However, parsing the RDB file directly from the socket may mean that we have +# to flush the contents of the current database before the full rdb was +# received. For this reason we have the following options: +# # "disabled" - Don't use diskless load (store the rdb file to the disk first) # "on-empty-db" - Use diskless load only when it is completely safe. # "swapdb" - Keep a copy of the current db contents in RAM while parsing From 920cbf051a2229f4f5f3e45812275d6eae8d3141 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 28 Jul 2019 14:33:57 +0100 Subject: [PATCH 041/320] Updating resident memory request impl on FreeBSD. --- src/zmalloc.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index 5e6010278..fd8bb6938 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -294,6 +294,26 @@ size_t zmalloc_get_rss(void) { return t_info.resident_size; } +#elif defined(__FreeBSD__) +#include +#include +#include +#include + +size_t zmalloc_get_rss(void) { + struct kinfo_proc info; + size_t infolen = sizeof(info); + int mib[4]; + mib[0] = CTL_KERN; + mib[1] = KERN_PROC; + mib[2] = KERN_PROC_PID; + mib[3] = getpid(); + + if (sysctl(mib, 4, &info, &infolen, NULL, 0) == 0) + return (size_t)info.ki_rssize; + + return 0L; +} #else size_t zmalloc_get_rss(void) { /* If we can't get the RSS in an OS-specific way for this system just From c6d5347a22079c3d93d02b59a63262395e69f92c Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 29 Jul 2019 18:11:52 -0400 Subject: [PATCH 042/320] Fix HLL corruption bug --- src/hyperloglog.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index e01ea6042..e0557f985 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -701,6 +701,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { first += span; } if (span == 0) return -1; /* Invalid format. */ + if (span >= end) return -1; /* Invalid format. */ next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; if (next >= end) next = NULL; From fe6d9e4c954753ef7dadc17510fee95b69edcca3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 30 Jul 2019 11:20:51 +0200 Subject: [PATCH 043/320] emptyDbGeneric(): call signalFlushDb() before deleting the keys. This was broken since a refactoring performed recently by myself. --- src/db.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 3d8b844e8..95eaf04e9 100644 --- a/src/db.c +++ b/src/db.c @@ -353,6 +353,11 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } + /* Make sure the WATCHed keys are affected by the FLUSH* commands. + * Note that we need to call the function while the keys are still + * there. */ + signalFlushedDb(dbnum); + int startdb, enddb; if (dbnum == -1) { startdb = 0; @@ -378,7 +383,6 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( } } if (dbnum == -1) flushSlaveKeysWithExpireList(); - signalFlushedDb(dbnum); return removed; } From 9fa285523c2070d59e0b992ecf43bb98f8b5c448 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 15:11:57 +0300 Subject: [PATCH 044/320] Avoid diskelss-load if modules did not declare they handle read errors --- src/module.c | 47 +++++++++++++++++++++++++++++++++++------------ src/redismodule.h | 10 ++++------ src/replication.c | 6 +++++- src/server.h | 3 +-- 4 files changed, 45 insertions(+), 21 deletions(-) diff --git a/src/module.c b/src/module.c index 68952b1e0..36384c018 100644 --- a/src/module.c +++ b/src/module.c @@ -51,6 +51,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int options; /* Moduile options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -771,6 +772,19 @@ long long RM_Milliseconds(void) { return mstime(); } +/* Set flags defining capabilities or behavior bit flags. + * + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS: + * Generally, modules don't need to bother with this, as the process will just + * terminate if a read error happens, however, setting this flag would allow + * repl-diskless-load to work if enabled. + * The module should use RedisModule_IsIOError after reads, before using the + * data that was read, and in case of error, propagate it upwards, and also be + * able to release the partially populated value and all it's allocations. */ +void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { + ctx->module->options = options; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -3143,7 +3157,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->flags & REDISMODULE_HANDLE_IO_ERRORS) { + if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } @@ -3157,19 +3171,29 @@ void moduleRDBLoadError(RedisModuleIO *io) { exit(1); } -/* Set flags defining capabilities or behavior */ -void RM_SetIOFlags(RedisModuleIO *io, int flags) { - io->flags = flags; -} +/* Returns 0 if there's at least one registered data type that did not declare + * REDISMODULE_OPTIONS_HANDLE_IO_ERRORS, in which case diskless loading should + * be avoided since it could cause data loss. */ +int moduleAllDatatypesHandleErrors() { + dictIterator *di = dictGetIterator(modules); + dictEntry *de; -/* Get flags which were set by RedisModule_SetIOFlags */ -int RM_GetIOFlags(RedisModuleIO *io) { - return io->flags; + while ((de = dictNext(di)) != NULL) { + struct RedisModule *module = dictGetVal(de); + if (listLength(module->types) && + !(module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS)) + { + dictReleaseIterator(di); + return 0; + } + } + dictReleaseIterator(di); + return 1; } /* Returns true if any previous IO API failed. - * for Load* APIs the REDISMODULE_HANDLE_IO_ERRORS flag must be set with - * RediModule_SetIOFlags first. */ + * for Load* APIs the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag must be set with + * RediModule_SetModuleOptions first. */ int RM_IsIOError(RedisModuleIO *io) { return io->error; } @@ -5434,9 +5458,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeSetValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); - REGISTER_API(GetIOFlags); - REGISTER_API(SetIOFlags); REGISTER_API(IsIOError); + REGISTER_API(SetModuleOptions); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index 154b5b405..f0f27c067 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -140,8 +140,8 @@ typedef uint64_t RedisModuleTimerID; /* Do filter RedisModule_Call() commands initiated by module itself. */ #define REDISMODULE_CMDFILTER_NOSELF (1<<0) -/* Declare that the module can handle errors with RedisModule_SetIOFlags */ -#define REDISMODULE_HANDLE_IO_ERRORS (1<<0) +/* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ +#define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) /* ------------------------- End of common defines ------------------------ */ @@ -274,9 +274,8 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); -void REDISMODULE_API_FUNC(RedisModule_SetIOFlags)(RedisModuleIO *io, int flags); -int REDISMODULE_API_FUNC(RedisModule_GetIOFlags)(RedisModuleIO *io); int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -450,9 +449,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeSetValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); - REDISMODULE_GET_API(SetIOFlags); - REDISMODULE_GET_API(GetIOFlags); REDISMODULE_GET_API(IsIOError); + REDISMODULE_GET_API(SetModuleOptions); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..14ae2f96c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1115,8 +1115,12 @@ void restartAOFAfterSYNC() { static int useDisklessLoad() { /* compute boolean decision to use diskless load */ - return server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || + int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); + /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ + if (enabled && !moduleAllDatatypesHandleErrors()) + enabled = 0; + return enabled; } /* Helper function for readSyncBulkPayload() to make backups of the current diff --git a/src/server.h b/src/server.h index 9957c3b5c..5991cfa6c 100644 --- a/src/server.h +++ b/src/server.h @@ -599,7 +599,6 @@ typedef struct RedisModuleIO { * 2 (current version with opcodes annotation). */ struct RedisModuleCtx *ctx; /* Optional context, see RM_GetContextFromIO()*/ struct redisObject *key; /* Optional name of key processed */ - int flags; /* flags declaring capabilities or behavior */ } RedisModuleIO; /* Macro to initialize an IO context. Note that the 'ver' field is populated @@ -612,7 +611,6 @@ typedef struct RedisModuleIO { iovar.ver = 0; \ iovar.key = keyptr; \ iovar.ctx = NULL; \ - iovar.flags = 0; \ } while(0); /* This is a structure used to export DEBUG DIGEST capabilities to Redis @@ -1530,6 +1528,7 @@ void moduleAcquireGIL(void); void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); +int moduleAllDatatypesHandleErrors(); /* Utils */ long long ustime(void); From 037ac27e41004deaca686f206b01cd4799ce493b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 21 Jul 2019 18:18:11 +0300 Subject: [PATCH 045/320] Add test for module diskless short reads --- tests/modules/testrdb.c | 13 ++++++- tests/unit/moduleapi/testrdb.tcl | 62 +++++++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index 415497a2f..d73c8bfd3 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -15,6 +15,8 @@ RedisModuleString *after_str = NULL; void *testrdb_type_load(RedisModuleIO *rdb, int encver) { int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return NULL; assert(count==1); assert(encver==1); RedisModuleString *str = RedisModule_LoadString(rdb); @@ -57,6 +59,8 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, before_str); before_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) before_str = RedisModule_LoadString(rdb); } else { @@ -64,14 +68,19 @@ int testrdb_aux_load(RedisModuleIO *rdb, int encver, int when) { RedisModule_FreeString(ctx, after_str); after_str = NULL; int count = RedisModule_LoadSigned(rdb); + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; if (count) after_str = RedisModule_LoadString(rdb); } + if (RedisModule_IsIOError(rdb)) + return REDISMODULE_ERR; return REDISMODULE_OK; } void testrdb_type_free(void *value) { - RedisModule_FreeString(NULL, (RedisModuleString*)value); + if (value) + RedisModule_FreeString(NULL, (RedisModuleString*)value); } int testrdb_set_before(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) @@ -171,6 +180,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_Init(ctx,"testrdb",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; + RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS); + if (argc > 0) RedisModule_StringToLongLong(argv[0], &conf_aux_count); diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index 22201a08e..c72570002 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -56,7 +56,67 @@ tags "modules" { } } + tags {repl} { + test {diskless loading short read with module} { + start_server [list overrides [list loadmodule "$testmodule"]] { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] - # TODO: test short read handling + # Set master and replica to use diskless replication + $master config set repl-diskless-sync yes + $master config set rdbcompression no + $replica config set repl-diskless-load swapdb + for {set k 0} {$k < 30} {incr k} { + r testrdb.set.key key$k [string repeat A [expr {int(rand()*1000000)}]] + } + # Start the replication process... + $master config set repl-diskless-sync-delay 0 + $replica replicaof $master_host $master_port + + # kill the replication at various points + set attempts 3 + if {$::accurate} { set attempts 10 } + for {set i 0} {$i < $attempts} {incr i} { + # wait for the replica to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + wait_for_log_message -1 "*Loading DB in memory*" 5 2000 1 + + # add some additional random sleep so that we kill the master on a different place each time + after [expr {int(rand()*100)}] + + # kill the replica connection on the master + set killed [$master client kill type replica] + + if {[catch { + set res [wait_for_log_message -1 "*Internal error in RDB*" 5 100 10] + if {$::verbose} { + puts $res + } + }]} { + puts "failed triggering short read" + # force the replica to try another full sync + $master client kill type replica + $master set asdf asdf + # the side effect of resizing the backlog is that it is flushed (16k is the min size) + $master config set repl-backlog-size [expr {16384 + $i}] + } + # wait for loading to stop (fail) + wait_for_condition 100 10 { + [s -1 loading] eq 0 + } else { + fail "Replica didn't disconnect" + } + } + # enable fast shutdown + $master config set rdb-key-save-delay 0 + } + } + } + } } From 4b9c5d8536a4f5e11b0129cb32fc78913975be7d Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 30 Jul 2019 16:32:58 +0300 Subject: [PATCH 046/320] Log message when modules prevent diskless-load --- src/replication.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 14ae2f96c..614aaec83 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1118,8 +1118,11 @@ static int useDisklessLoad() { int enabled = server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB || (server.repl_diskless_load == REPL_DISKLESS_LOAD_WHEN_DB_EMPTY && dbTotalServerKeyCount()==0); /* Check all modules handle read errors, otherwise it's not safe to use diskless load. */ - if (enabled && !moduleAllDatatypesHandleErrors()) + if (enabled && !moduleAllDatatypesHandleErrors()) { + serverLog(LL_WARNING, + "Skipping diskless-load because there are modules that don't handle read errors."); enabled = 0; + } return enabled; } From 6637823a4f899162141350dd58f201eb9854c3f2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 10:36:23 +0200 Subject: [PATCH 047/320] HyperLogLog: fix the fix of a corruption bug. --- src/hyperloglog.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index e0557f985..5d4afaa25 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -700,8 +700,7 @@ int hllSparseSet(robj *o, long index, uint8_t count) { p += oplen; first += span; } - if (span == 0) return -1; /* Invalid format. */ - if (span >= end) return -1; /* Invalid format. */ + if (span == 0 || p >= end) return -1; /* Invalid format. */ next = HLL_SPARSE_IS_XZERO(p) ? p+2 : p+1; if (next >= end) next = NULL; From 203a7762dcff800d73932907fd11968342d0d9ca Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 12:03:10 +0200 Subject: [PATCH 048/320] Make EMBSTR case of #6261 more obvious. --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index b9ff0a88b..fb011d311 100644 --- a/src/object.c +++ b/src/object.c @@ -472,7 +472,7 @@ robj *tryObjectEncoding(robj *o) { o->encoding = OBJ_ENCODING_INT; o->ptr = (void*) value; return o; - } else { + } else if (o->encoding == OBJ_ENCODING_EMBSTR) { decrRefCount(o); return createStringObjectFromLongLongForValue(value); } From b8f4ed24482d7a40fe96d49b1d47e9707c65a351 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 31 Jul 2019 19:04:29 +0200 Subject: [PATCH 049/320] Fix regression causing EXEC to appear in the slow log. This was recently introduced with PR #6266. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 9de604315..5b3b434df 100644 --- a/src/server.c +++ b/src/server.c @@ -3200,7 +3200,7 @@ void call(client *c, int flags) { /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ - if (flags & CMD_CALL_SLOWLOG && !(c->flags & CMD_SKIP_SLOWLOG)) { + if (flags & CMD_CALL_SLOWLOG && !(c->cmd->flags & CMD_SKIP_SLOWLOG)) { char *latency_event = (c->cmd->flags & CMD_FAST) ? "fast-command" : "command"; latencyAddSampleIfNeeded(latency_event,duration/1000); From 89dc16f01d159551df27060f47d536f77f84bef4 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 2 Aug 2019 17:17:19 +0800 Subject: [PATCH 050/320] networking: flushSlavesOutputBuffers bugfix --- src/networking.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/networking.c b/src/networking.c index 7976caf29..a39fd73d8 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2468,7 +2468,6 @@ void flushSlavesOutputBuffers(void) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = listNodeValue(ln); - int events; /* Note that the following will not flush output buffers of slaves * in STATE_ONLINE but having put_online_on_ack set to true: in this @@ -2476,9 +2475,8 @@ void flushSlavesOutputBuffers(void) { * of put_online_on_ack is to postpone the moment it is installed. * This is what we want since slaves in this state should not receive * writes before the first ACK. */ - events = aeGetFileEvents(server.el,slave->fd); - if (events & AE_WRITABLE && - slave->replstate == SLAVE_STATE_ONLINE && + if (slave->replstate == SLAVE_STATE_ONLINE && + !slave->repl_put_online_on_ack && clientHasPendingReplies(slave)) { writeToClient(slave->fd,slave,0); From 766550a7c6ad6fba5e1ea99fcbfdfcd2f5496181 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 2 Aug 2019 20:13:16 +0200 Subject: [PATCH 051/320] tracking_collisions.c: initial skeleton. ... of a program to just test the hashing functions collisions on a 24 bit output with strings that are very likely Redis key names, and names of a kind that are particularly prone to collisions. --- utils/tracking_collisions.c | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 utils/tracking_collisions.c diff --git a/utils/tracking_collisions.c b/utils/tracking_collisions.c new file mode 100644 index 000000000..b996452e7 --- /dev/null +++ b/utils/tracking_collisions.c @@ -0,0 +1,40 @@ +/* This is a small program used in order to understand the collison rate + * of CRC64 (ISO version) VS other stronger hashing functions in the context + * of hashing keys for the Redis "tracking" feature (client side caching + * assisted by the server). + * + * The program attempts to hash keys with common names in the form of + * + * prefix: + * + * And counts the resulting collisons generated in the 24 bits of output + * needed for the tracking feature invalidation table (16 millions + entries) + * + * -------------------------------------------------------------------------- + * + * Copyright (C) 2019 Salvatore Sanfilippo + * This code is released under the BSD 2 clause license. + */ + +#include +#include +#include +#include + +#define TABLE_SIZE (1<<24) +int Table[TABLE_SIZE]; + +/* Test the hashing function provided as callback and return the + * number of collisions found. */ +unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { + unsigned long collisions = 0; + memset(Table,0,sizeof(Table)); + char *prefixes[] = {"object", "message", "user", NULL}; + return collisions; +} + +int main(void) { + printf("SHA1 : %lu\n", testHashingFunction(sha1Hash)); + printf("CRC64: %lu\n", testHashingFunction(crc64Hash)); + return 0; +} From 7cd02bbab9c34e09874b969d7d54d3232f077ba1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 2 Aug 2019 20:24:27 +0200 Subject: [PATCH 052/320] tracking_collisions.c: sha1 + crc64 implemented. --- utils/tracking_collisions.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/utils/tracking_collisions.c b/utils/tracking_collisions.c index b996452e7..cd64b36c5 100644 --- a/utils/tracking_collisions.c +++ b/utils/tracking_collisions.c @@ -10,6 +10,11 @@ * And counts the resulting collisons generated in the 24 bits of output * needed for the tracking feature invalidation table (16 millions + entries) * + * Compile with: + * + * cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c + * ./a.out + * * -------------------------------------------------------------------------- * * Copyright (C) 2019 Salvatore Sanfilippo @@ -20,16 +25,47 @@ #include #include #include +#include "../src/crc64.h" +#include "../src/sha1.h" #define TABLE_SIZE (1<<24) int Table[TABLE_SIZE]; +uint64_t crc64Hash(char *key, size_t len) { + return crc64(0,(unsigned char*)key,len); +} + +uint64_t sha1Hash(char *key, size_t len) { + SHA1_CTX ctx; + unsigned char hash[20]; + + SHA1Init(&ctx); + SHA1Update(&ctx,(unsigned char*)key,len); + SHA1Final(hash,&ctx); + uint64_t hash64; + memcpy(&hash64,hash,sizeof(hash64)); + return hash64; +} + /* Test the hashing function provided as callback and return the * number of collisions found. */ unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) { unsigned long collisions = 0; memset(Table,0,sizeof(Table)); char *prefixes[] = {"object", "message", "user", NULL}; + for (int i = 0; prefixes[i] != NULL; i++) { + for (int j = 0; j < TABLE_SIZE/2; j++) { + char keyname[128]; + size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d", + prefixes[i],j); + uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE; + if (Table[bucket]) { + collisions++; + } else { + Table[bucket] = 1; + } + } + } return collisions; } From e87768e8ca3bf3d01e296bf88c2dd833689f7a4b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 5 Aug 2019 17:38:15 +0200 Subject: [PATCH 053/320] Replication: clarify why repl_put_online_on_ack exists at all. --- src/replication.c | 42 +++++++++++++++++++++++++++++++++--------- src/server.h | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/replication.c b/src/replication.c index 26e7cf8f0..d6646c9ef 100644 --- a/src/replication.c +++ b/src/replication.c @@ -823,7 +823,9 @@ void replconfCommand(client *c) { c->repl_ack_time = server.unixtime; /* If this was a diskless replication, we need to really put * the slave online when the first ACK is received (which - * confirms slave is online and ready to get more data). */ + * confirms slave is online and ready to get more data). This + * allows for simpler and less CPU intensive EOF detection + * when streaming RDB files. */ if (c->repl_put_online_on_ack && c->replstate == SLAVE_STATE_ONLINE) putSlaveOnline(c); /* Note: this command does not reply anything! */ @@ -842,18 +844,20 @@ void replconfCommand(client *c) { addReply(c,shared.ok); } -/* This function puts a slave in the online state, and should be called just - * after a slave received the RDB file for the initial synchronization, and +/* This function puts a replica in the online state, and should be called just + * after a replica received the RDB file for the initial synchronization, and * we are finally ready to send the incremental stream of commands. * * It does a few things: * - * 1) Put the slave in ONLINE state (useless when the function is called - * because state is already ONLINE but repl_put_online_on_ack is true). + * 1) Put the slave in ONLINE state. Note that the function may also be called + * for a replicas that are already in ONLINE state, but having the flag + * repl_put_online_on_ack set to true: we still have to install the write + * handler in that case. This function will take care of that. * 2) Make sure the writable event is re-installed, since calling the SYNC * command disables it, so that we can accumulate output buffer without - * sending it to the slave. - * 3) Update the count of good slaves. */ + * sending it to the replica. + * 3) Update the count of "good replicas". */ void putSlaveOnline(client *slave) { slave->replstate = SLAVE_STATE_ONLINE; slave->repl_put_online_on_ack = 0; @@ -965,11 +969,31 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { serverLog(LL_NOTICE, "Streamed RDB transfer with replica %s succeeded (socket). Waiting for REPLCONF ACK from slave to enable streaming", replicationGetSlaveName(slave)); - /* Note: we wait for a REPLCONF ACK message from slave in + /* Note: we wait for a REPLCONF ACK message from the replica in * order to really put it online (install the write handler * so that the accumulated data can be transferred). However * we change the replication state ASAP, since our slave - * is technically online now. */ + * is technically online now. + * + * So things work like that: + * + * 1. We end trasnferring the RDB file via socket. + * 2. The replica is put ONLINE but the write handler + * is not installed. + * 3. The replica however goes really online, and pings us + * back via REPLCONF ACK commands. + * 4. Now we finally install the write handler, and send + * the buffers accumulated so far to the replica. + * + * But why we do that? Because the replica, when we stream + * the RDB directly via the socket, must detect the RDB + * EOF (end of file), that is a special random string at the + * end of the RDB (for streamed RDBs we don't know the length + * in advance). Detecting such final EOF string is much + * simpler and less CPU intensive if no more data is sent + * after such final EOF. So we don't want to glue the end of + * the RDB trasfer with the start of the other replication + * data. */ slave->replstate = SLAVE_STATE_ONLINE; slave->repl_put_online_on_ack = 1; slave->repl_ack_time = server.unixtime; /* Timeout otherwise. */ diff --git a/src/server.h b/src/server.h index 13e596836..d003ce59a 100644 --- a/src/server.h +++ b/src/server.h @@ -848,7 +848,7 @@ typedef struct client { uint64_t flags; /* Client flags: CLIENT_* macros. */ int authenticated; /* Needed when the default user requires auth. */ int replstate; /* Replication state if this is a slave. */ - int repl_put_online_on_ack; /* Install slave write handler on ACK. */ + int repl_put_online_on_ack; /* Install slave write handler on first ACK. */ int repldbfd; /* Replication DB file descriptor. */ off_t repldboff; /* Replication DB file offset. */ off_t repldbsize; /* Replication DB file size. */ From f8b7c18d345b59ef47a0c0ab1a0f172caa671fa3 Mon Sep 17 00:00:00 2001 From: Diego Bendersky Date: Mon, 5 Aug 2019 17:35:50 -0300 Subject: [PATCH 054/320] pass len to alloc in clusterManagerMigrateKeysInReply --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index e363a2795..904339857 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -3222,7 +3222,7 @@ static redisReply *clusterManagerMigrateKeysInReply(clusterManagerNode *source, redisReply *entry = reply->element[i]; size_t idx = i + offset; assert(entry->type == REDIS_REPLY_STRING); - argv[idx] = (char *) sdsnew(entry->str); + argv[idx] = (char *) sdsnewlen(entry->str, entry->len); argv_len[idx] = entry->len; if (dots) dots[i] = '.'; } From 5973b475cb8753b9f0019086e6470c1bce079ea7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 7 Aug 2019 13:05:33 +0300 Subject: [PATCH 055/320] RM_ReplyWithCString was missing registration --- src/module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.c b/src/module.c index 275ee4344..ab614c529 100644 --- a/src/module.c +++ b/src/module.c @@ -5389,6 +5389,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplyWithString); REGISTER_API(ReplyWithStringBuffer); + REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); From f1c8492bca8f4c76fd26a5b77f9bd1c0b3dfd460 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 8 Aug 2019 14:53:36 +0300 Subject: [PATCH 056/320] fix error handling on config parsing of repl-diskless-load --- src/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.c b/src/config.c index a72df2e78..090fc818d 100644 --- a/src/config.c +++ b/src/config.c @@ -438,6 +438,7 @@ void loadServerConfigFromString(char *config) { server.repl_diskless_load = configEnumGetValue(repl_diskless_load_enum,argv[1]); if (server.repl_diskless_load == INT_MIN) { err = "argument must be 'disabled', 'on-empty-db', 'swapdb' or 'flushdb'"; + goto loaderr; } } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { server.repl_diskless_sync_delay = atoi(argv[1]); From d4c24a3067aabf820c42d2a27664bdff5ffd55f2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 18 Aug 2019 09:41:45 +0300 Subject: [PATCH 057/320] Module INFO, add support for dict fields, rename API to have common prefix --- src/module.c | 106 +++++++++++++++++++++++++----- src/redismodule.h | 28 ++++---- tests/modules/infotest.c | 26 +++++--- tests/unit/moduleapi/infotest.tcl | 7 +- 4 files changed, 129 insertions(+), 38 deletions(-) diff --git a/src/module.c b/src/module.c index 0e19c11e2..c48b2f045 100644 --- a/src/module.c +++ b/src/module.c @@ -43,9 +43,10 @@ typedef struct RedisModuleInfoCtx { struct RedisModule *module; sds requested_section; - sds info; /* info string we collected so far */ - int sections; /* number of sections we collected so far */ - int in_section; /* indication if we're in an active section or not */ + sds info; /* info string we collected so far */ + int sections; /* number of sections we collected so far */ + int in_section; /* indication if we're in an active section or not */ + int in_dict_field; /* indication that we're curreintly appending to a dict */ } RedisModuleInfoCtx; typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); @@ -4704,12 +4705,18 @@ int RM_DictCompare(RedisModuleDictIter *di, const char *op, RedisModuleString *k * Modules Info fields * -------------------------------------------------------------------------- */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); + /* Used to start a new section, before adding any fields. the section name will * be prefixed by "_" and must only include A-Z,a-z,0-9. * When return value is REDISMODULE_ERR, the section should and will be skipped. */ -int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { +int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + /* proceed only if: * 1) no section was requested (emit all) * 2) the module name was requested (emit all) @@ -4729,12 +4736,48 @@ int RM_AddInfoSection(RedisModuleInfoCtx *ctx, char *name) { return REDISMODULE_OK; } +/* Starts a dict field, similar to the ones in INFO KEYSPACE. Use normal + * RedisModule_InfoAddField* functions to add the items to this field, and + * terminate with RedisModule_InfoEndDictField. */ +int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { + if (!ctx->in_section) + return REDISMODULE_ERR; + /* Implicitly end dicts, instead of returning an error which is likely un checked. */ + if (ctx->in_dict_field) + RM_InfoEndDictField(ctx); + ctx->info = sdscatprintf(ctx->info, + "%s_%s:", + ctx->module->name, + name); + ctx->in_dict_field = 1; + return REDISMODULE_OK; +} + +/* Ends a dict field, see RedisModule_InfoBeginDictField */ +int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { + if (!ctx->in_dict_field) + return REDISMODULE_ERR; + /* trim the last ',' if found. */ + if (ctx->info[sdslen(ctx->info)-1]==',') + sdsIncrLen(ctx->info, -1); + ctx->info = sdscatprintf(ctx->info, "\r\n"); + ctx->in_dict_field = 0; + return REDISMODULE_OK; +} + /* Used by RedisModuleInfoFunc to add info fields. * Each field will be automatically prefixed by "_". * Field names or values must not include \r\n of ":" */ -int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { +int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%s,", + field, + (sds)value->ptr); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%s\r\n", ctx->module->name, @@ -4743,9 +4786,16 @@ int RM_AddInfoFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin return REDISMODULE_OK; } -int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { +int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%s,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%s\r\n", ctx->module->name, @@ -4754,9 +4804,16 @@ int RM_AddInfoFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { return REDISMODULE_OK; } -int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { +int RM_InfoAddFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%.17g,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%.17g\r\n", ctx->module->name, @@ -4765,9 +4822,16 @@ int RM_AddInfoFieldDouble(RedisModuleInfoCtx *ctx, char *field, double value) { return REDISMODULE_OK; } -int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { +int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%lld,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%lld\r\n", ctx->module->name, @@ -4776,9 +4840,16 @@ int RM_AddInfoFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu return REDISMODULE_OK; } -int RM_AddInfoFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { +int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long long value) { if (!ctx->in_section) return REDISMODULE_ERR; + if (ctx->in_dict_field) { + ctx->info = sdscatprintf(ctx->info, + "%s=%llu,", + field, + value); + return REDISMODULE_OK; + } ctx->info = sdscatprintf(ctx->info, "%s_%s:%llu\r\n", ctx->module->name, @@ -4802,6 +4873,9 @@ sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections continue; RedisModuleInfoCtx info_ctx = {module, section, info, sections, 0}; module->info_cb(&info_ctx, for_crash_report); + /* Implicitly end dicts (no way to handle errors, and we must add the newline). */ + if (info_ctx.in_dict_field) + RM_InfoEndDictField(&info_ctx); info = info_ctx.info; sections = info_ctx.sections; } @@ -5614,10 +5688,12 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CommandFilterArgReplace); REGISTER_API(CommandFilterArgDelete); REGISTER_API(RegisterInfoFunc); - REGISTER_API(AddInfoSection); - REGISTER_API(AddInfoFieldString); - REGISTER_API(AddInfoFieldCString); - REGISTER_API(AddInfoFieldDouble); - REGISTER_API(AddInfoFieldLongLong); - REGISTER_API(AddInfoFieldULongLong); + REGISTER_API(InfoAddSection); + REGISTER_API(InfoBeginDictField); + REGISTER_API(InfoEndDictField); + REGISTER_API(InfoAddFieldString); + REGISTER_API(InfoAddFieldCString); + REGISTER_API(InfoAddFieldDouble); + REGISTER_API(InfoAddFieldLongLong); + REGISTER_API(InfoAddFieldULongLong); } diff --git a/src/redismodule.h b/src/redismodule.h index 1feb14a68..26690c17c 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -320,12 +320,14 @@ RedisModuleString *REDISMODULE_API_FUNC(RedisModule_DictPrev)(RedisModuleCtx *ct int REDISMODULE_API_FUNC(RedisModule_DictCompareC)(RedisModuleDictIter *di, const char *op, void *key, size_t keylen); int REDISMODULE_API_FUNC(RedisModule_DictCompare)(RedisModuleDictIter *di, const char *op, RedisModuleString *key); int REDISMODULE_API_FUNC(RedisModule_RegisterInfoFunc)(RedisModuleCtx *ctx, RedisModuleInfoFunc cb); -int REDISMODULE_API_FUNC(RedisModule_AddInfoSection)(RedisModuleInfoCtx *ctx, char *name); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); -int REDISMODULE_API_FUNC(RedisModule_AddInfoFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddSection)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_InfoBeginDictField)(RedisModuleInfoCtx *ctx, char *name); +int REDISMODULE_API_FUNC(RedisModule_InfoEndDictField)(RedisModuleInfoCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldString)(RedisModuleInfoCtx *ctx, char *field, RedisModuleString *value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ctx, char *field, char *value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); +int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -500,12 +502,14 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(DictCompare); REDISMODULE_GET_API(DictCompareC); REDISMODULE_GET_API(RegisterInfoFunc); - REDISMODULE_GET_API(AddInfoSection); - REDISMODULE_GET_API(AddInfoFieldString); - REDISMODULE_GET_API(AddInfoFieldCString); - REDISMODULE_GET_API(AddInfoFieldDouble); - REDISMODULE_GET_API(AddInfoFieldLongLong); - REDISMODULE_GET_API(AddInfoFieldULongLong); + REDISMODULE_GET_API(InfoAddSection); + REDISMODULE_GET_API(InfoBeginDictField); + REDISMODULE_GET_API(InfoEndDictField); + REDISMODULE_GET_API(InfoAddFieldString); + REDISMODULE_GET_API(InfoAddFieldCString); + REDISMODULE_GET_API(InfoAddFieldDouble); + REDISMODULE_GET_API(InfoAddFieldLongLong); + REDISMODULE_GET_API(InfoAddFieldULongLong); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index d53ea2126..1a75d1606 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -3,19 +3,25 @@ #include void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { - RedisModule_AddInfoSection(ctx, "Spanish"); - RedisModule_AddInfoFieldCString(ctx, "uno", "one"); - RedisModule_AddInfoFieldLongLong(ctx, "dos", 2); + RedisModule_InfoAddSection(ctx, "Spanish"); + RedisModule_InfoAddFieldCString(ctx, "uno", "one"); + RedisModule_InfoAddFieldLongLong(ctx, "dos", 2); - RedisModule_AddInfoSection(ctx, "Italian"); - RedisModule_AddInfoFieldLongLong(ctx, "due", 2); - RedisModule_AddInfoFieldDouble(ctx, "tre", 3.3); + RedisModule_InfoAddSection(ctx, "Italian"); + RedisModule_InfoAddFieldLongLong(ctx, "due", 2); + RedisModule_InfoAddFieldDouble(ctx, "tre", 3.3); + + RedisModule_InfoAddSection(ctx, "keyspace"); + RedisModule_InfoBeginDictField(ctx, "db0"); + RedisModule_InfoAddFieldLongLong(ctx, "keys", 3); + RedisModule_InfoAddFieldLongLong(ctx, "expires", 1); + RedisModule_InfoEndDictField(ctx); if (for_crash_report) { - RedisModule_AddInfoSection(ctx, "Klingon"); - RedisModule_AddInfoFieldCString(ctx, "one", "wa’"); - RedisModule_AddInfoFieldCString(ctx, "two", "cha’"); - RedisModule_AddInfoFieldCString(ctx, "three", "wej"); + RedisModule_InfoAddSection(ctx, "Klingon"); + RedisModule_InfoAddFieldCString(ctx, "one", "wa’"); + RedisModule_InfoAddFieldCString(ctx, "two", "cha’"); + RedisModule_InfoAddFieldCString(ctx, "three", "wej"); } } diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 143f9050a..7f756f0e6 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -48,6 +48,11 @@ start_server {tags {"modules"}} { field $info infotest_uno } {one} - # TODO: test crash report. + test {module info dict} { + set info [r info infotest_keyspace] + set keyspace [field $info infotest_db0] + set keys [scan [regexp -inline {keys\=([\d]*)} $keyspace] keys=%d] + } {3} + # TODO: test crash report. } From 8319ffb34d83355fcc73eb13ecac1951f124d5df Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 18 Aug 2019 10:01:57 +0300 Subject: [PATCH 058/320] Module INFO, support default section for simple modules --- src/module.c | 5 ++++- tests/modules/infotest.c | 3 +++ tests/unit/moduleapi/infotest.tcl | 7 ++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index c48b2f045..046b9bc86 100644 --- a/src/module.c +++ b/src/module.c @@ -4709,9 +4709,12 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); /* Used to start a new section, before adding any fields. the section name will * be prefixed by "_" and must only include A-Z,a-z,0-9. + * NULL or empty string indicates the default section (only ) is used. * When return value is REDISMODULE_ERR, the section should and will be skipped. */ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { - sds full_name = sdscatprintf(sdsdup(ctx->module->name), "_%s", name); + sds full_name = sdsdup(ctx->module->name); + if (name != NULL && strlen(name) > 0) + full_name = sdscatprintf(full_name, "_%s", name); /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index 1a75d1606..d28410932 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -3,6 +3,9 @@ #include void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { + RedisModule_InfoAddSection(ctx, ""); + RedisModule_InfoAddFieldLongLong(ctx, "global", -2); + RedisModule_InfoAddSection(ctx, "Spanish"); RedisModule_InfoAddFieldCString(ctx, "uno", "one"); RedisModule_InfoAddFieldLongLong(ctx, "dos", 2); diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 7f756f0e6..a64d729f3 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -14,12 +14,14 @@ start_server {tags {"modules"}} { set info [r info all] # info all does not contain modules assert { ![string match "*Spanish*" $info] } + assert { ![string match "*infotest*" $info] } assert { [string match "*used_memory*" $info] } } test {module info everything} { set info [r info everything] # info everything contains all default sections, but not ones for crash report + assert { [string match "*infotest_global*" $info] } assert { [string match "*Spanish*" $info] } assert { [string match "*Italian*" $info] } assert { [string match "*used_memory*" $info] } @@ -31,6 +33,7 @@ start_server {tags {"modules"}} { set info [r info modules] # info all does not contain modules assert { [string match "*Spanish*" $info] } + assert { [string match "*infotest_global*" $info] } assert { ![string match "*used_memory*" $info] } } @@ -39,12 +42,14 @@ start_server {tags {"modules"}} { # info all does not contain modules assert { [string match "*Spanish*" $info] } assert { ![string match "*used_memory*" $info] } - } + field $info infotest_global + } {-2} test {module info one section} { set info [r info INFOTEST_SPANISH] assert { ![string match "*used_memory*" $info] } assert { ![string match "*Italian*" $info] } + assert { ![string match "*infotest_global*" $info] } field $info infotest_uno } {one} From 8bf7c6d789986fd8a8823a1fb41eed8e498cbc59 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 25 Aug 2019 10:11:48 +0300 Subject: [PATCH 059/320] Modlue fork is killed when the parent exists --- src/module.c | 26 +++++++++++++++----------- src/server.c | 6 ++++++ src/server.h | 1 + 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/module.c b/src/module.c index 03cc36702..6f3be61af 100644 --- a/src/module.c +++ b/src/module.c @@ -5086,20 +5086,11 @@ int RM_ExitFromChild(int retcode) return REDISMODULE_OK; } -/* Can be used to kill the forked child process from the parent process. - * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ +void TerminateModuleForkChild(int wait) { int statloc; - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; - /* Kill module child, wait for child exit. */ serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); - if (kill(server.module_child_pid,SIGUSR1) != -1) { + if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { while(wait3(&statloc,0,NULL) != server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ @@ -5108,6 +5099,19 @@ int RM_KillForkChild(int child_pid) moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); +} + +/* Can be used to kill the forked child process from the parent process. + * child_pid whould be the return value of RedisModule_Fork. */ +int RM_KillForkChild(int child_pid) +{ + /* No module child? return. */ + if (server.module_child_pid == -1) return REDISMODULE_ERR; + /* Make sure the module knows the pid it wants to kill (not trying to + * randomly kill other module's forks) */ + if (server.module_child_pid != child_pid) return REDISMODULE_ERR; + /* Kill module child, wait for child exit. */ + TerminateModuleForkChild(1); return REDISMODULE_OK; } diff --git a/src/server.c b/src/server.c index 675b638d6..31418e01b 100644 --- a/src/server.c +++ b/src/server.c @@ -3551,6 +3551,12 @@ int prepareForShutdown(int flags) { killRDBChild(); } + /* Kill module child if there is one. */ + if (server.module_child_pid != -1) { + serverLog(LL_WARNING,"There is a module fork child. Killing it!"); + TerminateModuleForkChild(0); + } + if (server.aof_state != AOF_OFF) { /* Kill the AOF saving child as the AOF we already have may be longer * but contains the full dataset anyway. */ diff --git a/src/server.h b/src/server.h index deaaf2636..7aa4bc2b7 100644 --- a/src/server.h +++ b/src/server.h @@ -1532,6 +1532,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); +void TerminateModuleForkChild(int wait); /* Utils */ long long ustime(void); From e6c2020a558988b6bd9c729f572eb4fe73e51b7a Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Sat, 31 Aug 2019 07:47:11 +0000 Subject: [PATCH 060/320] src/debug.c do not support aarch64 dump utcontext, so add the context of aarch64. The content comes from the definition of the sigcontext and tested on my aarch64 server. sigcontext defined at the linux kernel code: arch/arm64/include/uapi/asm/sigcontext.h --- src/debug.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..595e45f72 100644 --- a/src/debug.c +++ b/src/debug.c @@ -1110,6 +1110,33 @@ void logRegisters(ucontext_t *uc) { (unsigned long) uc->uc_mcontext.mc_cs ); logStackContent((void**)uc->uc_mcontext.mc_rsp); +#elif defined(__aarch64__) /* Linux AArch64 */ + serverLog(LL_WARNING, + "\n" + "X18:%016lx X19:%016lx\nX20:%016lx X21:%016lx\n" + "X22:%016lx X23:%016lx\nX24:%016lx X25:%016lx\n" + "X26:%016lx X27:%016lx\nX28:%016lx X29:%016lx\n" + "X30:%016lx\n" + "pc:%016lx sp:%016lx\npstate:%016lx fault_address:%016lx\n", + (unsigned long) uc->uc_mcontext.regs[18], + (unsigned long) uc->uc_mcontext.regs[19], + (unsigned long) uc->uc_mcontext.regs[20], + (unsigned long) uc->uc_mcontext.regs[21], + (unsigned long) uc->uc_mcontext.regs[22], + (unsigned long) uc->uc_mcontext.regs[23], + (unsigned long) uc->uc_mcontext.regs[24], + (unsigned long) uc->uc_mcontext.regs[25], + (unsigned long) uc->uc_mcontext.regs[26], + (unsigned long) uc->uc_mcontext.regs[27], + (unsigned long) uc->uc_mcontext.regs[28], + (unsigned long) uc->uc_mcontext.regs[29], + (unsigned long) uc->uc_mcontext.regs[30], + (unsigned long) uc->uc_mcontext.pc, + (unsigned long) uc->uc_mcontext.sp, + (unsigned long) uc->uc_mcontext.pstate, + (unsigned long) uc->uc_mcontext.fault_address + ); + logStackContent((void**)uc->uc_mcontext.sp); #else serverLog(LL_WARNING, " Dumping of registers not supported for this OS/arch"); From 11228bd0fb1ca5fc743dba2342fbe7cdf409c779 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Aug 2019 14:40:09 +0200 Subject: [PATCH 061/320] Improve comment in flushSlavesOutputBuffers(). --- src/networking.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7976caf29..9ff2afd8d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2475,7 +2475,8 @@ void flushSlavesOutputBuffers(void) { * case the writable event is never installed, since the purpose * of put_online_on_ack is to postpone the moment it is installed. * This is what we want since slaves in this state should not receive - * writes before the first ACK. */ + * writes before the first ACK (to know the reason, grep for this + * flag in this file). */ events = aeGetFileEvents(server.el,slave->fd); if (events & AE_WRITABLE && slave->replstate == SLAVE_STATE_ONLINE && From 3b40a8858770286b217ebc1fdd97a6a9950bae46 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Aug 2019 14:46:21 +0200 Subject: [PATCH 062/320] More strict checks and better comments in flushSlaveOutputBuffers(). Related to #6296. --- src/networking.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/networking.c b/src/networking.c index 7bdd3c474..7555ca77d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2468,15 +2468,26 @@ void flushSlavesOutputBuffers(void) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = listNodeValue(ln); + int events = aeGetFileEvents(server.el,slave->fd); + int can_receive_writes = (events & AE_WRITABLE) || + (slave->flags & CLIENT_PENDING_WRITE); - /* Note that the following will not flush output buffers of slaves - * in STATE_ONLINE but having put_online_on_ack set to true: in this - * case the writable event is never installed, since the purpose - * of put_online_on_ack is to postpone the moment it is installed. - * This is what we want since slaves in this state should not receive - * writes before the first ACK (to know the reason, grep for this - * flag in this file). */ + /* We don't want to send the pending data to the replica in a few + * cases: + * + * 1. For some reason there is neither the write handler installed + * nor the client is flagged as to have pending writes: for some + * reason this replica may not be set to receive data. This is + * just for the sake of defensive programming. + * + * 2. The put_online_on_ack flag is true. To know why we don't want + * to send data to the replica in this case, please grep for the + * flag for this flag. + * + * 3. Obviously if the slave is not ONLINE. + */ if (slave->replstate == SLAVE_STATE_ONLINE && + can_receive_writes && !slave->repl_put_online_on_ack && clientHasPendingReplies(slave)) { From 4ff08c865715f1c379e9c0e119caaa3afcfa1ed6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Sep 2019 11:41:16 +0200 Subject: [PATCH 063/320] Cluster: abort loading nodes data if vars arguments are unbalanced. See for reference PR #6337. Thanks to @git-hulk for spotting this. --- src/cluster.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cluster.c b/src/cluster.c index e22222700..a2615fdc0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -138,6 +138,7 @@ int clusterLoadConfig(char *filename) { /* Handle the special "vars" line. Don't pretend it is the last * line even if it actually is when generated by Redis. */ if (strcasecmp(argv[0],"vars") == 0) { + if (!(argc % 2)) goto fmterr; for (j = 1; j < argc; j += 2) { if (strcasecmp(argv[j],"currentEpoch") == 0) { server.cluster->currentEpoch = From 2e1e170aad04f27d66551c95f1a2ef41d4930874 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Sep 2019 12:50:47 +0200 Subject: [PATCH 064/320] RESP3: fix cases of NULL reported instead of empty aggregate. --- src/server.c | 10 ++++++++++ src/server.h | 2 +- src/t_hash.c | 4 ++-- src/t_list.c | 4 ++-- src/t_set.c | 10 +++++----- src/t_zset.c | 18 +++++++++--------- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/server.c b/src/server.c index 5b3b434df..fb308e5f8 100644 --- a/src/server.c +++ b/src/server.c @@ -2161,6 +2161,16 @@ void createSharedObjects(void) { shared.nullarray[2] = createObject(OBJ_STRING,sdsnew("*-1\r\n")); shared.nullarray[3] = createObject(OBJ_STRING,sdsnew("_\r\n")); + shared.emptymap[0] = NULL; + shared.emptymap[1] = NULL; + shared.emptymap[2] = createObject(OBJ_STRING,sdsnew("*0\r\n")); + shared.emptymap[3] = createObject(OBJ_STRING,sdsnew("%0\r\n")); + + shared.emptyset[0] = NULL; + shared.emptyset[1] = NULL; + shared.emptyset[2] = createObject(OBJ_STRING,sdsnew("*0\r\n")); + shared.emptyset[3] = createObject(OBJ_STRING,sdsnew("~0\r\n")); + for (j = 0; j < PROTO_SHARED_SELECT_CMDS; j++) { char dictid_str[64]; int dictid_len; diff --git a/src/server.h b/src/server.h index d003ce59a..31d253066 100644 --- a/src/server.h +++ b/src/server.h @@ -897,7 +897,7 @@ struct moduleLoadQueueEntry { struct sharedObjectsStruct { robj *crlf, *ok, *err, *emptybulk, *czero, *cone, *pong, *space, - *colon, *queued, *null[4], *nullarray[4], + *colon, *queued, *null[4], *nullarray[4], *emptymap[4], *emptyset[4], *emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, diff --git a/src/t_hash.c b/src/t_hash.c index bc70e4051..e6ed33819 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -772,8 +772,8 @@ void genericHgetallCommand(client *c, int flags) { hashTypeIterator *hi; int length, count = 0; - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL - || checkType(c,o,OBJ_HASH)) return; + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymap[c->resp])) + == NULL || checkType(c,o,OBJ_HASH)) return; /* We return a map if the user requested keys and values, like in the * HGETALL case. Otherwise to use a flat array makes more sense. */ diff --git a/src/t_list.c b/src/t_list.c index 54e4959b9..601ea8899 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -402,7 +402,7 @@ void lrangeCommand(client *c) { if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL || checkType(c,o,OBJ_LIST)) return; llen = listTypeLength(o); @@ -414,7 +414,7 @@ void lrangeCommand(client *c) { /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } if (end >= llen) end = llen-1; diff --git a/src/t_set.c b/src/t_set.c index 05d9ee243..abbf82fde 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -418,10 +418,10 @@ void spopWithCountCommand(client *c) { if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.null[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; - /* If count is zero, serve an empty multibulk ASAP to avoid special + /* If count is zero, serve an empty set ASAP to avoid special * cases later. */ if (count == 0) { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); return; } @@ -632,13 +632,13 @@ void srandmemberWithCountCommand(client *c) { uniq = 0; } - if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) + if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyset[c->resp])) == NULL || checkType(c,set,OBJ_SET)) return; size = setTypeSize(set); /* If count is zero, serve it ASAP to avoid special cases later. */ if (count == 0) { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); return; } @@ -813,7 +813,7 @@ void sinterGenericCommand(client *c, robj **setkeys, } addReply(c,shared.czero); } else { - addReplyNull(c); + addReply(c,shared.emptyset[c->resp]); } return; } diff --git a/src/t_zset.c b/src/t_zset.c index 2680e76a9..ea6f4b848 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2426,7 +2426,7 @@ void zrangeGenericCommand(client *c, int reverse) { return; } - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) return; /* Sanitize indexes. */ @@ -2438,7 +2438,7 @@ void zrangeGenericCommand(client *c, int reverse) { /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } if (end >= llen) end = llen-1; @@ -2574,7 +2574,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { } /* Ok, lookup the key and get the range */ - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) return; if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { @@ -2594,7 +2594,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (eptr == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } @@ -2661,7 +2661,7 @@ void genericZrangebyscoreCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (ln == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } @@ -2919,7 +2919,7 @@ void genericZrangebylexCommand(client *c, int reverse) { } /* Ok, lookup the key and get the range */ - if ((zobj = lookupKeyReadOrReply(c,key,shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c,key,shared.emptyarray)) == NULL || checkType(c,zobj,OBJ_ZSET)) { zslFreeLexRange(&range); @@ -2942,7 +2942,7 @@ void genericZrangebylexCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (eptr == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); zslFreeLexRange(&range); return; } @@ -3006,7 +3006,7 @@ void genericZrangebylexCommand(client *c, int reverse) { /* No "first" element in the specified interval. */ if (ln == NULL) { - addReplyNull(c); + addReply(c,shared.emptyarray); zslFreeLexRange(&range); return; } @@ -3160,7 +3160,7 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey /* No candidate for zpopping, return empty. */ if (!zobj) { - addReplyNull(c); + addReply(c,shared.emptyarray); return; } From 9658ff71bf3064780c71646a24925576ef153433 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 13:01:07 +0200 Subject: [PATCH 065/320] Rio: fix flag name, function is never used btw. Thanks to @tnclong for reporting the problem. --- src/rio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rio.h b/src/rio.h index 6199bb039..eb7a05748 100644 --- a/src/rio.h +++ b/src/rio.h @@ -150,7 +150,7 @@ static inline int rioGetReadError(rio *r) { /* Like rioGetReadError() but for write errors. */ static inline int rioGetWriteError(rio *r) { - return (r->flags & RIO_FLAG_READ_ERROR) != 0; + return (r->flags & RIO_FLAG_WRITE_ERROR) != 0; } static inline void rioClearErrors(rio *r) { From 40c8f23fe768f0c369087e9d4567cf8f791da525 Mon Sep 17 00:00:00 2001 From: zhudacai 00228490 Date: Wed, 4 Sep 2019 12:14:25 +0000 Subject: [PATCH 066/320] The aarch64 architecture is support normal memory unaligned accesses, so add the UNALIGNED_LE_CPU to the aarch64 . --- src/siphash.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/siphash.c b/src/siphash.c index 6b9419031..357741132 100644 --- a/src/siphash.c +++ b/src/siphash.c @@ -58,7 +58,8 @@ int siptlw(int c) { /* Test of the CPU is Little Endian and supports not aligned accesses. * Two interesting conditions to speedup the function that happen to be * in most of x86 servers. */ -#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) +#if defined(__X86_64__) || defined(__x86_64__) || defined (__i386__) \ + || defined (__aarch64__) || defined (__arm64__) #define UNALIGNED_LE_CPU #endif From 4383958317ed3d0d285dba98950944801e448ba0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 17:20:37 +0200 Subject: [PATCH 067/320] AOF: be future-proof and close the file pointer. Currently useless but we release the fake client, so better to do a full cleanup. Thanks to @TomMD reporting this in #6353. --- src/aof.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/aof.c b/src/aof.c index ae9c4bb68..7237cdfbc 100644 --- a/src/aof.c +++ b/src/aof.c @@ -862,6 +862,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ readerr: /* Read error. If feof(fp) is true, fall through to unexpected EOF. */ if (!feof(fp)) { if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno)); exit(1); } @@ -892,11 +893,13 @@ uxeof: /* Unexpected AOF end of file. */ } } if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Unexpected end of file reading the append only file. You can: 1) Make a backup of your AOF file, then use ./redis-check-aof --fix . 2) Alternatively you can set the 'aof-load-truncated' configuration option to yes and restart the server."); exit(1); fmterr: /* Format error. */ if (fakeClient) freeFakeClient(fakeClient); /* avoid valgrind warning */ + fclose(fp); serverLog(LL_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix "); exit(1); } From 1f813b3723d39c8f21b6b17b31d14d73beb59486 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2019 17:55:46 +0200 Subject: [PATCH 068/320] redis-cli: always report server errors on read errors. Before this commit we may have not consumer buffers when a read error is encountered. Such buffers may contain errors that are important clues for the user: for instance a protocol error in the payload we send in pipe mode will cause the server to abort the connection. If the user does not get the protocol error, debugging what is happening can be a nightmare. This commit fixes issue #3756. --- src/redis-cli.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index e363a2795..db53fc7d8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -6724,6 +6724,7 @@ static void pipeMode(void) { /* Handle the readable state: we can read replies from the server. */ if (mask & AE_READABLE) { ssize_t nread; + int read_error = 0; /* Read from socket and feed the hiredis reader. */ do { @@ -6731,7 +6732,8 @@ static void pipeMode(void) { if (nread == -1 && errno != EAGAIN && errno != EINTR) { fprintf(stderr, "Error reading from the server: %s\n", strerror(errno)); - exit(1); + read_error = 1; + break; } if (nread > 0) { redisReaderFeed(reader,ibuf,nread); @@ -6764,6 +6766,11 @@ static void pipeMode(void) { freeReplyObject(reply); } } while(reply); + + /* Abort on read errors. We abort here because it is important + * to consume replies even after a read error: this way we can + * show a potential problem to the user. */ + if (read_error) exit(1); } /* Handle the writable state: we can send protocol to the server. */ From 3984b4fdd9ea122f7b8fbe4a56fdaeb973ccab5b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Sep 2019 13:05:57 +0200 Subject: [PATCH 069/320] Fix handleClientsBlockedOnKeys() names in comments. --- src/blocked.c | 2 +- src/t_list.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 1db657869..9c3907fd4 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -592,7 +592,7 @@ void unblockClientWaitingData(client *c) { * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * - * The list will be finally processed by handleClientsBlockedOnLists() */ + * The list will be finally processed by handleClientsBlockedOnKeys() */ void signalKeyAsReady(redisDb *db, robj *key) { readyList *rl; diff --git a/src/t_list.c b/src/t_list.c index 601ea8899..9bbd61de3 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -606,7 +606,7 @@ void rpoplpushCommand(client *c) { * Blocking POP operations *----------------------------------------------------------------------------*/ -/* This is a helper function for handleClientsBlockedOnLists(). It's work +/* This is a helper function for handleClientsBlockedOnKeys(). It's work * is to serve a specific client (receiver) that is blocked on 'key' * in the context of the specified 'db', doing the following: * From 68e1809589d17eebe766a77fd9e5e8c90afff9f3 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Sep 2019 14:11:37 +0300 Subject: [PATCH 070/320] Fix to module aux data rdb format for backwards compatibility with old check-rdb When implementing the code that saves and loads these aux fields we used rdb format that was added for that in redis 5.0, but then we added the 'when' field which meant that the old redis-check-rdb won't be able to skip these. this fix adds an opcode as if that 'when' is part of the module data. --- src/rdb.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 4e00fad67..e4dfc46b7 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1114,7 +1114,12 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) { if (retval == -1) return -1; io.bytes += retval; - /* write the 'when' so that we can provide it on loading */ + /* write the 'when' so that we can provide it on loading. add a UINT opcode + * for backwards compatibility, everything after the MT needs to be prefixed + * by an opcode. */ + retval = rdbSaveLen(rdb,RDB_MODULE_OPCODE_UINT); + if (retval == -1) return -1; + io.bytes += retval; retval = rdbSaveLen(rdb,when); if (retval == -1) return -1; io.bytes += retval; @@ -2132,8 +2137,11 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { * Such data can be potentially be stored both before and after the * RDB keys-values section. */ uint64_t moduleid = rdbLoadLen(rdb,NULL); + int when_opcode = rdbLoadLen(rdb,NULL); int when = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) goto eoferr; + if (when_opcode != RDB_MODULE_OPCODE_UINT) + rdbReportReadError("bad when_opcode"); moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; moduleTypeNameByID(name,moduleid); From f701f7c007665cebf4aa9350a1dcc42da6498434 Mon Sep 17 00:00:00 2001 From: Doug Nelson Date: Thu, 5 Sep 2019 16:25:06 +0100 Subject: [PATCH 071/320] Typo fixes in API documentation --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c529..85674ba0d 100644 --- a/src/module.c +++ b/src/module.c @@ -2389,7 +2389,7 @@ int RM_HashSet(RedisModuleKey *key, int flags, ...) { * * REDISMODULE_HASH_EXISTS: instead of setting the value of the field * expecting a RedisModuleString pointer to pointer, the function just - * reports if the field esists or not and expects an integer pointer + * reports if the field exists or not and expects an integer pointer * as the second element of each pair. * * Example of REDISMODULE_HASH_CFIELD: @@ -3288,7 +3288,7 @@ RedisModuleString *RM_LoadString(RedisModuleIO *io) { * RedisModule_Realloc() or RedisModule_Free(). * * The size of the string is stored at '*lenptr' if not NULL. - * The returned string is not automatically NULL termianted, it is loaded + * The returned string is not automatically NULL terminated, it is loaded * exactly as it was stored inisde the RDB file. */ char *RM_LoadStringBuffer(RedisModuleIO *io, size_t *lenptr) { return moduleLoadString(io,1,lenptr); From 62e74777458bcfbf51f081a2158926be3cc18fca Mon Sep 17 00:00:00 2001 From: suntiawnen Date: Fri, 6 Sep 2019 12:01:44 +0800 Subject: [PATCH 072/320] fix rdb function rdbLoadIntegerObject comment --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index e4dfc46b7..d11fb93f7 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -260,7 +260,7 @@ int rdbEncodeInteger(long long value, unsigned char *enc) { /* Loads an integer-encoded object with the specified encoding type "enctype". * The returned value changes according to the flags, see - * rdbGenerincLoadStringObject() for more info. */ + * rdbGenericLoadStringObject() for more info. */ void *rdbLoadIntegerObject(rio *rdb, int enctype, int flags, size_t *lenptr) { int plain = flags & RDB_LOAD_PLAIN; int sds = flags & RDB_LOAD_SDS; From c98af3a5507e75a70111bcf74e3529e3ed2ec3a6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Sep 2019 12:24:26 +0200 Subject: [PATCH 073/320] handleClientsBlockedOnKeys() refactoring. --- src/blocked.c | 403 ++++++++++++++++++++++++++------------------------ 1 file changed, 208 insertions(+), 195 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 9c3907fd4..867f03de6 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -229,6 +229,207 @@ void disconnectAllBlockedClients(void) { } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a list key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnListKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_LIST) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + robj *dstkey = receiver->bpop.target; + int where = (receiver->lastcmd && + receiver->lastcmd->proc == blpopCommand) ? + LIST_HEAD : LIST_TAIL; + robj *value = listTypePop(o,where); + + if (value) { + /* Protect receiver->bpop.target, that will be + * freed by the next unblockClient() + * call. */ + if (dstkey) incrRefCount(dstkey); + unblockClient(receiver); + + if (serveClientBlockedOnList(receiver, + rl->key,dstkey,rl->db,value, + where) == C_ERR) + { + /* If we failed serving the client we need + * to also undo the POP operation. */ + listTypePush(o,value,where); + } + + if (dstkey) decrRefCount(dstkey); + decrRefCount(value); + } else { + break; + } + } + } + + if (listTypeLength(o) == 0) { + dbDelete(rl->db,rl->key); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); + } + /* We don't call signalModifiedKey() as it was already called + * when an element was pushed on the list. */ +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a sorted set key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnSortedSetKey(robj *o, readyList *rl) { + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + unsigned long zcard = zsetLength(o); + + while(numclients-- && zcard) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (receiver->btype != BLOCKED_ZSET) { + /* Put at the tail, so that at the next call + * we'll not run into it again. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + continue; + } + + int where = (receiver->lastcmd && + receiver->lastcmd->proc == bzpopminCommand) + ? ZSET_MIN : ZSET_MAX; + unblockClient(receiver); + genericZpopCommand(receiver,&rl->key,1,where,1,NULL); + zcard--; + + /* Replicate the command. */ + robj *argv[2]; + struct redisCommand *cmd = where == ZSET_MIN ? + server.zpopminCommand : + server.zpopmaxCommand; + argv[0] = createStringObject(cmd->name,strlen(cmd->name)); + argv[1] = rl->key; + incrRefCount(rl->key); + propagate(cmd,receiver->db->id, + argv,2,PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(argv[0]); + decrRefCount(argv[1]); + } + } +} + +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * when there may be clients blocked on a stream key, and there may be new + * data to fetch (the key is ready). */ +void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { + dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); + stream *s = o->ptr; + + /* We need to provide the new data arrived on the stream + * to all the clients that are waiting for an offset smaller + * than the current top item. */ + if (de) { + list *clients = dictGetVal(de); + listNode *ln; + listIter li; + listRewind(clients,&li); + + while((ln = listNext(&li))) { + client *receiver = listNodeValue(ln); + if (receiver->btype != BLOCKED_STREAM) continue; + streamID *gt = dictFetchValue(receiver->bpop.keys, + rl->key); + + /* If we blocked in the context of a consumer + * group, we need to resolve the group and update the + * last ID the client is blocked for: this is needed + * because serving other clients in the same consumer + * group will alter the "last ID" of the consumer + * group, and clients blocked in a consumer group are + * always blocked for the ">" ID: we need to deliver + * only new messages and avoid unblocking the client + * otherwise. */ + streamCG *group = NULL; + if (receiver->bpop.xread_group) { + group = streamLookupCG(s, + receiver->bpop.xread_group->ptr); + /* If the group was not found, send an error + * to the consumer. */ + if (!group) { + addReplyError(receiver, + "-NOGROUP the consumer group this client " + "was blocked on no longer exists"); + unblockClient(receiver); + continue; + } else { + *gt = group->last_id; + } + } + + if (streamCompareID(&s->last_id, gt) > 0) { + streamID start = *gt; + start.seq++; /* Can't overflow, it's an uint64_t */ + + /* Lookup the consumer for the group, if any. */ + streamConsumer *consumer = NULL; + int noack = 0; + + if (group) { + consumer = streamLookupConsumer(group, + receiver->bpop.xread_consumer->ptr, + 1); + noack = receiver->bpop.xread_group_noack; + } + + /* Emit the two elements sub-array consisting of + * the name of the stream and the data we + * extracted from it. Wrapped in a single-item + * array, since we have just one key. */ + if (receiver->resp == 2) { + addReplyArrayLen(receiver,1); + addReplyArrayLen(receiver,2); + } else { + addReplyMapLen(receiver,1); + } + addReplyBulk(receiver,rl->key); + + streamPropInfo pi = { + rl->key, + receiver->bpop.xread_group + }; + streamReplyWithRange(receiver,s,&start,NULL, + receiver->bpop.xread_count, + 0, group, consumer, noack, &pi); + + /* Note that after we unblock the client, 'gt' + * and other receiver->bpop stuff are no longer + * valid, so we must do the setup above before + * this call. */ + unblockClient(receiver); + } + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -271,202 +472,14 @@ void handleClientsBlockedOnKeys(void) { /* Serve clients blocked on list key. */ robj *o = lookupKeyWrite(rl->db,rl->key); - if (o != NULL && o->type == OBJ_LIST) { - dictEntry *de; - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - - while(numclients--) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_LIST) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - robj *dstkey = receiver->bpop.target; - int where = (receiver->lastcmd && - receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; - robj *value = listTypePop(o,where); - - if (value) { - /* Protect receiver->bpop.target, that will be - * freed by the next unblockClient() - * call. */ - if (dstkey) incrRefCount(dstkey); - unblockClient(receiver); - - if (serveClientBlockedOnList(receiver, - rl->key,dstkey,rl->db,value, - where) == C_ERR) - { - /* If we failed serving the client we need - * to also undo the POP operation. */ - listTypePush(o,value,where); - } - - if (dstkey) decrRefCount(dstkey); - decrRefCount(value); - } else { - break; - } - } - } - - if (listTypeLength(o) == 0) { - dbDelete(rl->db,rl->key); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",rl->key,rl->db->id); - } - /* We don't call signalModifiedKey() as it was already called - * when an element was pushed on the list. */ - } - - /* Serve clients blocked on sorted set key. */ - else if (o != NULL && o->type == OBJ_ZSET) { - dictEntry *de; - - /* We serve clients in the same order they blocked for - * this key, from the first blocked to the last. */ - de = dictFind(rl->db->blocking_keys,rl->key); - if (de) { - list *clients = dictGetVal(de); - int numclients = listLength(clients); - unsigned long zcard = zsetLength(o); - - while(numclients-- && zcard) { - listNode *clientnode = listFirst(clients); - client *receiver = clientnode->value; - - if (receiver->btype != BLOCKED_ZSET) { - /* Put at the tail, so that at the next call - * we'll not run into it again. */ - listDelNode(clients,clientnode); - listAddNodeTail(clients,receiver); - continue; - } - - int where = (receiver->lastcmd && - receiver->lastcmd->proc == bzpopminCommand) - ? ZSET_MIN : ZSET_MAX; - unblockClient(receiver); - genericZpopCommand(receiver,&rl->key,1,where,1,NULL); - zcard--; - - /* Replicate the command. */ - robj *argv[2]; - struct redisCommand *cmd = where == ZSET_MIN ? - server.zpopminCommand : - server.zpopmaxCommand; - argv[0] = createStringObject(cmd->name,strlen(cmd->name)); - argv[1] = rl->key; - incrRefCount(rl->key); - propagate(cmd,receiver->db->id, - argv,2,PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(argv[0]); - decrRefCount(argv[1]); - } - } - } - - /* Serve clients blocked on stream key. */ - else if (o != NULL && o->type == OBJ_STREAM) { - dictEntry *de = dictFind(rl->db->blocking_keys,rl->key); - stream *s = o->ptr; - - /* We need to provide the new data arrived on the stream - * to all the clients that are waiting for an offset smaller - * than the current top item. */ - if (de) { - list *clients = dictGetVal(de); - listNode *ln; - listIter li; - listRewind(clients,&li); - - while((ln = listNext(&li))) { - client *receiver = listNodeValue(ln); - if (receiver->btype != BLOCKED_STREAM) continue; - streamID *gt = dictFetchValue(receiver->bpop.keys, - rl->key); - - /* If we blocked in the context of a consumer - * group, we need to resolve the group and update the - * last ID the client is blocked for: this is needed - * because serving other clients in the same consumer - * group will alter the "last ID" of the consumer - * group, and clients blocked in a consumer group are - * always blocked for the ">" ID: we need to deliver - * only new messages and avoid unblocking the client - * otherwise. */ - streamCG *group = NULL; - if (receiver->bpop.xread_group) { - group = streamLookupCG(s, - receiver->bpop.xread_group->ptr); - /* If the group was not found, send an error - * to the consumer. */ - if (!group) { - addReplyError(receiver, - "-NOGROUP the consumer group this client " - "was blocked on no longer exists"); - unblockClient(receiver); - continue; - } else { - *gt = group->last_id; - } - } - - if (streamCompareID(&s->last_id, gt) > 0) { - streamID start = *gt; - start.seq++; /* Can't overflow, it's an uint64_t */ - - /* Lookup the consumer for the group, if any. */ - streamConsumer *consumer = NULL; - int noack = 0; - - if (group) { - consumer = streamLookupConsumer(group, - receiver->bpop.xread_consumer->ptr, - 1); - noack = receiver->bpop.xread_group_noack; - } - - /* Emit the two elements sub-array consisting of - * the name of the stream and the data we - * extracted from it. Wrapped in a single-item - * array, since we have just one key. */ - if (receiver->resp == 2) { - addReplyArrayLen(receiver,1); - addReplyArrayLen(receiver,2); - } else { - addReplyMapLen(receiver,1); - } - addReplyBulk(receiver,rl->key); - - streamPropInfo pi = { - rl->key, - receiver->bpop.xread_group - }; - streamReplyWithRange(receiver,s,&start,NULL, - receiver->bpop.xread_count, - 0, group, consumer, noack, &pi); - - /* Note that after we unblock the client, 'gt' - * and other receiver->bpop stuff are no longer - * valid, so we must do the setup above before - * this call. */ - unblockClient(receiver); - } - } - } + if (o != NULL) { + if (o->type == OBJ_LIST) + serveClientsBlockedOnListKey(o,rl); + else if (o->type == OBJ_ZSET) + serveClientsBlockedOnSortedSetKey(o,rl); + else if (o->type == OBJ_STREAM) + serveClientsBlockedOnStreamKey(o,rl); } /* Free this item. */ From e645c794cf654da8d13cf46e838b12f6ab9f5f94 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Sep 2019 19:42:10 +0200 Subject: [PATCH 074/320] ACL: protect MULTI/EXEC transactions after rules change. --- src/multi.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/multi.c b/src/multi.c index 71090d8ed..f885fa19c 100644 --- a/src/multi.c +++ b/src/multi.c @@ -175,7 +175,19 @@ void execCommand(client *c) { must_propagate = 1; } - call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + int acl_retval = ACLCheckCommandPerm(c); + if (acl_retval != ACL_OK) { + addReplyErrorFormat(c, + "-NOPERM ACLs rules changed between the moment the " + "transaction was accumulated and the EXEC call. " + "This command is no longer allowed for the " + "following reason: %s", + (acl_retval == ACL_DENIED_CMD) ? + "no permission to execute the command or subcommand" : + "no permission to touch the specified keys"); + } else { + call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); + } /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; From ecf8dc6ff6fed0a73e1588416db3fcfa9516ee79 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:21:34 +0200 Subject: [PATCH 075/320] ACL: add slightly modified version of sha256.c for password hashing. memory.h include removed, types substituted with stdint types. --- src/Makefile | 2 +- src/sha256.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/sha256.h | 35 ++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/sha256.c create mode 100644 src/sha256.h diff --git a/src/Makefile b/src/Makefile index b6cc69e2f..198d85cd5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -164,7 +164,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/sha256.c b/src/sha256.c new file mode 100644 index 000000000..d644d2d4e --- /dev/null +++ b/src/sha256.c @@ -0,0 +1,158 @@ +/********************************************************************* +* Filename: sha256.c +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Implementation of the SHA-256 hashing algorithm. + SHA-256 is one of the three algorithms in the SHA2 + specification. The others, SHA-384 and SHA-512, are not + offered in this implementation. + Algorithm specification can be found here: + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2withchangenotice.pdf + This implementation uses little endian byte order. +*********************************************************************/ + +/*************************** HEADER FILES ***************************/ +#include +#include +#include "sha256.h" + +/****************************** MACROS ******************************/ +#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b)))) +#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b)))) + +#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22)) +#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25)) +#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3)) +#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10)) + +/**************************** VARIABLES *****************************/ +static const WORD k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +/*********************** FUNCTION DEFINITIONS ***********************/ +void sha256_transform(SHA256_CTX *ctx, const BYTE data[]) +{ + WORD a, b, c, d, e, f, g, h, i, j, t1, t2, m[64]; + + for (i = 0, j = 0; i < 16; ++i, j += 4) + m[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | (data[j + 3]); + for ( ; i < 64; ++i) + m[i] = SIG1(m[i - 2]) + m[i - 7] + SIG0(m[i - 15]) + m[i - 16]; + + a = ctx->state[0]; + b = ctx->state[1]; + c = ctx->state[2]; + d = ctx->state[3]; + e = ctx->state[4]; + f = ctx->state[5]; + g = ctx->state[6]; + h = ctx->state[7]; + + for (i = 0; i < 64; ++i) { + t1 = h + EP1(e) + CH(e,f,g) + k[i] + m[i]; + t2 = EP0(a) + MAJ(a,b,c); + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + + ctx->state[0] += a; + ctx->state[1] += b; + ctx->state[2] += c; + ctx->state[3] += d; + ctx->state[4] += e; + ctx->state[5] += f; + ctx->state[6] += g; + ctx->state[7] += h; +} + +void sha256_init(SHA256_CTX *ctx) +{ + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; + ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; + ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; + ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; + ctx->state[7] = 0x5be0cd19; +} + +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len) +{ + WORD i; + + for (i = 0; i < len; ++i) { + ctx->data[ctx->datalen] = data[i]; + ctx->datalen++; + if (ctx->datalen == 64) { + sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +void sha256_final(SHA256_CTX *ctx, BYTE hash[]) +{ + WORD i; + + i = ctx->datalen; + + // Pad whatever data is left in the buffer. + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) + ctx->data[i++] = 0x00; + } + else { + ctx->data[i++] = 0x80; + while (i < 64) + ctx->data[i++] = 0x00; + sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + + // Append to the padding the total message's length in bits and transform. + ctx->bitlen += ctx->datalen * 8; + ctx->data[63] = ctx->bitlen; + ctx->data[62] = ctx->bitlen >> 8; + ctx->data[61] = ctx->bitlen >> 16; + ctx->data[60] = ctx->bitlen >> 24; + ctx->data[59] = ctx->bitlen >> 32; + ctx->data[58] = ctx->bitlen >> 40; + ctx->data[57] = ctx->bitlen >> 48; + ctx->data[56] = ctx->bitlen >> 56; + sha256_transform(ctx, ctx->data); + + // Since this implementation uses little endian byte ordering and SHA uses big endian, + // reverse all the bytes when copying the final state to the output hash. + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0x000000ff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0x000000ff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0x000000ff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0x000000ff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0x000000ff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0x000000ff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0x000000ff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0x000000ff; + } +} diff --git a/src/sha256.h b/src/sha256.h new file mode 100644 index 000000000..dc53ead2b --- /dev/null +++ b/src/sha256.h @@ -0,0 +1,35 @@ +/********************************************************************* +* Filename: sha256.h +* Author: Brad Conte (brad AT bradconte.com) +* Copyright: +* Disclaimer: This code is presented "as is" without any guarantees. +* Details: Defines the API for the corresponding SHA1 implementation. +*********************************************************************/ + +#ifndef SHA256_H +#define SHA256_H + +/*************************** HEADER FILES ***************************/ +#include +#include + +/****************************** MACROS ******************************/ +#define SHA256_BLOCK_SIZE 32 // SHA256 outputs a 32 byte digest + +/**************************** DATA TYPES ****************************/ +typedef uint8_t BYTE; // 8-bit byte +typedef uint32_t WORD; // 32-bit word + +typedef struct { + BYTE data[64]; + WORD datalen; + unsigned long long bitlen; + WORD state[8]; +} SHA256_CTX; + +/*********************** FUNCTION DECLARATIONS **********************/ +void sha256_init(SHA256_CTX *ctx); +void sha256_update(SHA256_CTX *ctx, const BYTE data[], size_t len); +void sha256_final(SHA256_CTX *ctx, BYTE hash[]); + +#endif // SHA256_H From 036c83d8ec94639491b5d29b6e072e618eead4e3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:33:22 +0200 Subject: [PATCH 076/320] ACL: SHA256 based password hashing function implemented. --- src/acl.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/acl.c b/src/acl.c index a2ee65dd0..aea4fe97f 100644 --- a/src/acl.c +++ b/src/acl.c @@ -28,6 +28,7 @@ */ #include "server.h" +#include "sha256.h" #include /* ============================================================================= @@ -139,6 +140,25 @@ int time_independent_strcmp(char *a, char *b) { return diff; /* If zero strings are the same. */ } +/* Given an SDS string, returns the SHA256 hex representation as a + * new SDS string. */ +sds ACLHashPassword(sds cleartext) { + SHA256_CTX ctx; + unsigned char hash[SHA256_BLOCK_SIZE]; + char hex[SHA256_BLOCK_SIZE*2]; + char *cset = "0123456789abcdef"; + + sha256_init(&ctx); + sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_final(&ctx,hash); + + for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { + hex[j*2] = cset[((hash[j]&0xF0)>>4)]; + hex[j*2+1] = cset[(hash[j]&0xF)]; + } + return sdsnewlen(hex,SHA256_BLOCK_SIZE*2); +} + /* ============================================================================= * Low level ACL API * ==========================================================================*/ From fa2ed64ba15fcd034c49c9c7e34006bfaed3998b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Sep 2019 12:54:57 +0200 Subject: [PATCH 077/320] ACL: store hashed passwords in memory. Note that this breaks API compatibility with Redis < 6: CONFIG GET requirepass Will no longer return a cleartext password as well, but the SHA256 hash of the password set. --- src/acl.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/acl.c b/src/acl.c index aea4fe97f..2cd729e77 100644 --- a/src/acl.c +++ b/src/acl.c @@ -142,14 +142,14 @@ int time_independent_strcmp(char *a, char *b) { /* Given an SDS string, returns the SHA256 hex representation as a * new SDS string. */ -sds ACLHashPassword(sds cleartext) { +sds ACLHashPassword(unsigned char *cleartext, size_t len) { SHA256_CTX ctx; unsigned char hash[SHA256_BLOCK_SIZE]; char hex[SHA256_BLOCK_SIZE*2]; char *cset = "0123456789abcdef"; sha256_init(&ctx); - sha256_update(&ctx,(unsigned char*)cleartext,sdslen(cleartext)); + sha256_update(&ctx,(unsigned char*)cleartext,len); sha256_final(&ctx,hash); for (int j = 0; j < SHA256_BLOCK_SIZE; j++) { @@ -721,13 +721,16 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); } else if (op[0] == '>') { - sds newpass = sdsnewlen(op+1,oplen-1); + sds newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ - if (ln == NULL) listAddNodeTail(u->passwords,newpass); + if (ln == NULL) + listAddNodeTail(u->passwords,newpass); + else + sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; } else if (op[0] == '<') { - sds delpass = sdsnewlen(op+1,oplen-1); + sds delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { @@ -744,7 +747,10 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { sds newpat = sdsnewlen(op+1,oplen-1); listNode *ln = listSearchKey(u->patterns,newpat); /* Avoid re-adding the same pattern multiple times. */ - if (ln == NULL) listAddNodeTail(u->patterns,newpat); + if (ln == NULL) + listAddNodeTail(u->patterns,newpat); + else + sdsfree(newpat); u->flags &= ~USER_FLAG_ALLKEYS; } else if (op[0] == '+' && op[1] != '@') { if (strchr(op,'|') == NULL) { @@ -899,11 +905,15 @@ int ACLCheckUserCredentials(robj *username, robj *password) { listIter li; listNode *ln; listRewind(u->passwords,&li); + sds hashed = ACLHashPassword(password->ptr,sdslen(password->ptr)); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - if (!time_independent_strcmp(password->ptr, thispass)) + if (!time_independent_strcmp(hashed, thispass)) { + sdsfree(hashed); return C_OK; + } } + sdsfree(hashed); /* If we reached this point, no password matched. */ errno = EINVAL; From 33e35c3562dcf58fce8cafeb6afb45f00fbd2e03 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:01:39 +0200 Subject: [PATCH 078/320] RESP3: implement lua.setresp(). --- src/scripting.c | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 032bfdf10..d5da14332 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -42,7 +42,7 @@ char *redisProtocolToLuaType_Int(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -132,9 +132,9 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '$': p = redisProtocolToLuaType_Bulk(lua,reply); break; case '+': p = redisProtocolToLuaType_Status(lua,reply); break; case '-': p = redisProtocolToLuaType_Error(lua,reply); break; - case '*': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '%': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; - case '~': p = redisProtocolToLuaType_MultiBulk(lua,reply,*p); break; + case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; } return p; } @@ -182,7 +182,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply) { return p+2; } -char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply, int atype) { +char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *p = strchr(reply+1,'\r'); long long mbulklen; int j = 0; @@ -859,6 +859,25 @@ int luaLogCommand(lua_State *lua) { return 0; } +/* redis.setresp() */ +int luaSetResp(lua_State *lua) { + int argc = lua_gettop(lua); + + if (argc != 1) { + lua_pushstring(lua, "redis.setresp() requires one argument."); + return lua_error(lua); + } + + int resp = lua_tonumber(lua,-argc); + if (resp != 2 && resp != 3) { + lua_pushstring(lua, "RESP version must be 2 or 3."); + return lua_error(lua); + } + + server.lua_client->resp = resp; + return 0; +} + /* --------------------------------------------------------------------------- * Lua engine initialization and reset. * ------------------------------------------------------------------------- */ @@ -986,6 +1005,11 @@ void scriptingInit(int setup) { lua_pushcfunction(lua,luaLogCommand); lua_settable(lua,-3); + /* redis.setresp */ + lua_pushstring(lua,"setresp"); + lua_pushcfunction(lua,luaSetResp); + lua_settable(lua,-3); + lua_pushstring(lua,"LOG_DEBUG"); lua_pushnumber(lua,LL_DEBUG); lua_settable(lua,-3); @@ -1379,8 +1403,9 @@ void evalGenericCommand(client *c, int evalsha) { luaSetGlobalArray(lua,"KEYS",c->argv+3,numkeys); luaSetGlobalArray(lua,"ARGV",c->argv+3+numkeys,c->argc-3-numkeys); - /* Select the right DB in the context of the Lua client */ + /* Set the Lua client database and protocol. */ selectDb(server.lua_client,c->db->id); + server.lua_client->resp = 2; /* Default is RESP2, scripts can change it. */ /* Set a hook in order to be able to stop the script execution if it * is running for too much time. From 1eecd5f2de75d69ddeab714501206d45b712262c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:19:10 +0200 Subject: [PATCH 079/320] RESP3: Lua debugger support for printing sets and maps. --- src/scripting.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index d5da14332..68c7ce006 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2078,6 +2078,8 @@ char *ldbRedisProtocolToHuman_Int(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2092,6 +2094,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '+': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '-': p = ldbRedisProtocolToHuman_Status(o,reply); break; case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; + case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; + case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; } return p; } @@ -2146,6 +2150,40 @@ char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Set(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"~(",2); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,")",1); + return p; +} + +char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + long long mbulklen; + int j = 0; + + string2ll(reply+1,p-reply-1,&mbulklen); + p += 2; + *o = sdscatlen(*o,"{",1); + for (j = 0; j < mbulklen; j++) { + p = ldbRedisProtocolToHuman(o,p); + *o = sdscatlen(*o," => ",4); + p = ldbRedisProtocolToHuman(o,p); + if (j != mbulklen-1) *o = sdscatlen(*o,",",1); + } + *o = sdscatlen(*o,"}",1); + return p; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From e563362f1ddcbbe0c7efdd6d772f80d8b2fc5ddb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Sep 2019 19:38:35 +0200 Subject: [PATCH 080/320] RESP3: Lua parsing should depend on lua client, not lua caller. We want all the scripts to run in RESP2 mode by default. It's up to the caller to switch to V3 using redis.setresp() if it is really needed. This way most scripts written for past Redis versions will continue to work with Redis >= 6 even if the client is in RESP3 mode. --- src/scripting.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 68c7ce006..ee37ac23f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -188,7 +188,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { int j = 0; string2ll(reply+1,p-reply-1,&mbulklen); - if (server.lua_caller->resp == 2 || atype == '*') { + if (server.lua_client->resp == 2 || atype == '*') { p += 2; if (mbulklen == -1) { lua_pushboolean(lua,0); @@ -200,7 +200,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { p = redisProtocolToLuaType(lua,p); lua_settable(lua,-3); } - } else if (server.lua_caller->resp == 3) { + } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays * follow the above RESP2 code path. */ p += 2; From af51468e4f2d4394b648cef8bf71255ee9efc72f Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sun, 15 Sep 2019 14:05:00 +0100 Subject: [PATCH 081/320] Getting region date per process in Darwin --- src/zmalloc.c | 12 ++++++++++++ src/zmalloc.h | 1 + 2 files changed, 13 insertions(+) diff --git a/src/zmalloc.c b/src/zmalloc.c index fd8bb6938..48f4b1e99 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -396,6 +396,18 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { } #else size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { +#if defined(__APPLE__) + struct proc_regioninfo pri; + if (proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri, PROC_PIDREGIONINFO_SIZE) == + PROC_PIDREGIONINFO_SIZE) { + if (!strcmp(field, "Private_Dirty")) { + return (size_t)pri.pri_pages_dirtied * 4096; + } else if (!strcmp(field, "Rss")) { + return (size_t)pri.pri_pages_resident * 4096; + } + } + return 0; +#endif ((void) field); ((void) pid); return 0; diff --git a/src/zmalloc.h b/src/zmalloc.h index 6fb19b046..4aa33509d 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -57,6 +57,7 @@ #elif defined(__APPLE__) #include +#include #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) malloc_size(p) #endif From 6184d97323c225b1fb213cbd2f2ff5e0b6dd89ca Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Sun, 15 Sep 2019 21:16:30 +0100 Subject: [PATCH 082/320] [add] improved performance of RM_ReplyWithSimpleString and RM_ReplyWithError by making usage addReplyProto instead of addReplySds --- src/module.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/module.c b/src/module.c index ab614c529..ad7e6864c 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,19 +1120,6 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } -/* Reply with an error or simple string (status message). Used to implement - * ReplyWithSimpleString() and ReplyWithError(). - * The function always returns REDISMODULE_OK. */ -int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - sds strmsg = sdsnewlen(prefix,1); - strmsg = sdscat(strmsg,msg); - strmsg = sdscatlen(strmsg,"\r\n",2); - addReplySds(c,strmsg); - return REDISMODULE_OK; -} - /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1148,7 +1135,13 @@ int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - return replyWithStatus(ctx,err,"-"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(err); + addReplyProto(c,"-",1); + addReplyProto(c,err,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1157,7 +1150,13 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - return replyWithStatus(ctx,msg,"+"); + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"+",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; } /* Reply with an array type of 'len' elements. However 'len' other calls From dde8e3f8f45020f7c004bfdd6dfb72124bdc5d06 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 11:49:42 +0200 Subject: [PATCH 083/320] RESP3: report set/map as nested tables to Lua. --- src/scripting.c | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index ee37ac23f..39f75abf0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -202,9 +202,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } } else if (server.lua_client->resp == 3) { /* Here we handle only Set and Map replies in RESP3 mode, since arrays - * follow the above RESP2 code path. */ + * follow the above RESP2 code path. Note that those are represented + * as a table with the "map" or "set" field populated with the actual + * table representing the set or the map type. */ p += 2; lua_newtable(lua); + lua_pushstring(lua,atype == '%' ? "map" : "set"); + lua_newtable(lua); for (j = 0; j < mbulklen; j++) { p = redisProtocolToLuaType(lua,p); if (atype == '%') { @@ -214,6 +218,7 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { } lua_settable(lua,-3); } + lua_settable(lua,-3); } return p; } @@ -310,6 +315,8 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { * Error are returned as a single element table with 'err' field. * Status replies are returned as single element table with 'ok' * field. */ + + /* Handle error reply. */ lua_pushstring(lua,"err"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -321,8 +328,9 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { lua_pop(lua,2); return; } + lua_pop(lua,1); /* Discard field name pushed before. */ - lua_pop(lua,1); + /* Handle status reply. */ lua_pushstring(lua,"ok"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -332,24 +340,25 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); lua_pop(lua,1); - } else { - void *replylen = addReplyDeferredLen(c); - int j = 1, mbulklen = 0; - - lua_pop(lua,1); /* Discard the 'ok' field value we popped */ - while(1) { - lua_pushnumber(lua,j++); - lua_gettable(lua,-2); - t = lua_type(lua,-1); - if (t == LUA_TNIL) { - lua_pop(lua,1); - break; - } - luaReplyToRedisReply(c, lua); - mbulklen++; - } - setDeferredArrayLen(c,replylen,mbulklen); + return; } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle the array reply. */ + void *replylen = addReplyDeferredLen(c); + int j = 1, mbulklen = 0; + while(1) { + lua_pushnumber(lua,j++); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNIL) { + lua_pop(lua,1); + break; + } + luaReplyToRedisReply(c, lua); + mbulklen++; + } + setDeferredArrayLen(c,replylen,mbulklen); break; default: addReplyNull(c); From 347f85bddb9050d27a747e4b32de3956cea4f6af Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:15:39 +0200 Subject: [PATCH 084/320] RESP3: handle map Lua -> Redis conversion. --- src/scripting.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 39f75abf0..cd55000d4 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -297,6 +297,8 @@ void luaSortArray(lua_State *lua) { * Lua reply to Redis reply conversion functions. * ------------------------------------------------------------------------- */ +/* Reply to client 'c' converting the top element in the Lua stack to a + * Redis reply. As a side effect the element is consumed from the stack. */ void luaReplyToRedisReply(client *c, lua_State *lua) { int t = lua_type(lua,-1); @@ -339,7 +341,29 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { sdsmapchars(ok,"\r\n"," ",2); addReplySds(c,sdscatprintf(sdsempty(),"+%s\r\n",ok)); sdsfree(ok); - lua_pop(lua,1); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + + /* Handle map reply. */ + lua_pushstring(lua,"map"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int maplen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, value */ + luaReplyToRedisReply(c, lua); /* Return value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + maplen++; + } + setDeferredMapLen(c,replylen,maplen); + lua_pop(lua,2); return; } lua_pop(lua,1); /* Discard field name pushed before. */ From 5303bf6890b8ce5479adc5a9c0cb7c045f1f6980 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 12:19:19 +0200 Subject: [PATCH 085/320] RESP3: handle set Lua -> Redis conversion. --- src/scripting.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index cd55000d4..2c90bb7ae 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -368,6 +368,28 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle set reply. */ + lua_pushstring(lua,"set"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TTABLE) { + int setlen = 0; + void *replylen = addReplyDeferredLen(c); + lua_pushnil(lua); /* Use nil to start iteration. */ + while (lua_next(lua,-2)) { + /* Stack now: table, key, true */ + lua_pop(lua,1); /* Discard the boolean value. */ + lua_pushvalue(lua,-1); /* Dup key before consuming. */ + luaReplyToRedisReply(c, lua); /* Return key. */ + /* Stack now: table, key. */ + setlen++; + } + setDeferredSetLen(c,replylen,setlen); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle the array reply. */ void *replylen = addReplyDeferredLen(c); int j = 1, mbulklen = 0; From 547f3a29ab41fa16d2c207d5cef2e7dd20469f67 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 17:49:40 +0200 Subject: [PATCH 086/320] RESP3: implement new NULL representation parsing in Lua. --- src/scripting.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 2c90bb7ae..564dad8ca 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -43,6 +43,7 @@ char *redisProtocolToLuaType_Bulk(lua_State *lua, char *reply); char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -135,6 +136,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '*': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; + case '_': p = redisProtocolToLuaType_Null(lua,reply); break; } return p; } @@ -223,6 +225,12 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { return p; } +char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,0); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From 65a2bc7807c70428cf2c6834f64ea608a7f75721 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:18:17 +0200 Subject: [PATCH 087/320] RESP3: change behavior of Lua returning true/false for RESP3. Here we introduce a change in the way we convert values from Lua to Redis when RESP3 is selected: this is possible without breaking the fact we can return directly what a command returned, because there is no Redis command in RESP2 that returns true or false to Lua, so the conversion in the case of RESP2 is totally arbitrary. When a script is written selecting RESP3 from Lua, it totally makes sense to change such behavior and return RESP3 true/false when Lua true/false is returned. --- src/scripting.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 564dad8ca..3ad228a67 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -315,7 +315,11 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { addReplyBulkCBuffer(c,(char*)lua_tostring(lua,-1),lua_strlen(lua,-1)); break; case LUA_TBOOLEAN: - addReply(c,lua_toboolean(lua,-1) ? shared.cone : shared.null[c->resp]); + if (server.lua_client->resp == 2) + addReply(c,lua_toboolean(lua,-1) ? shared.cone : + shared.null[c->resp]); + else + addReplyBool(c,lua_toboolean(lua,-1)); break; case LUA_TNUMBER: addReplyLongLong(c,(long long)lua_tonumber(lua,-1)); From ff27682d78103392c4a22be68a24756dca90527d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Sep 2019 18:36:16 +0200 Subject: [PATCH 088/320] RESP3: convert RESP3 null as Lua nil. Implement RESP3->Lua bools. --- src/scripting.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 3ad228a67..9ab1e7ab3 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -44,6 +44,7 @@ char *redisProtocolToLuaType_Status(lua_State *lua, char *reply); char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -137,6 +138,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); } return p; } @@ -227,7 +229,13 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { char *redisProtocolToLuaType_Null(lua_State *lua, char *reply) { char *p = strchr(reply+1,'\r'); - lua_pushboolean(lua,0); + lua_pushnil(lua); + return p+2; +} + +char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { + char *p = strchr(reply+1,'\r'); + lua_pushboolean(lua,tf == 't'); return p+2; } From 1fdefd825d2e08090a2dbda9c9d23f11ea679fe4 Mon Sep 17 00:00:00 2001 From: Okada Haruki Date: Tue, 17 Sep 2019 06:18:01 +0900 Subject: [PATCH 089/320] Fix typo --- src/hyperloglog.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 5d4afaa25..a44d15646 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1242,7 +1242,7 @@ void pfcountCommand(client *c) { if (o == NULL) continue; /* Assume empty HLL for non existing var.*/ if (isHLLObjectOrReply(c,o) != C_OK) return; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(registers,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); @@ -1329,7 +1329,7 @@ void pfmergeCommand(client *c) { hdr = o->ptr; if (hdr->encoding == HLL_DENSE) use_dense = 1; - /* Merge with this HLL with our 'max' HHL by setting max[i] + /* Merge with this HLL with our 'max' HLL by setting max[i] * to MAX(max[i],hll[i]). */ if (hllMerge(max,o) == C_ERR) { addReplySds(c,sdsnew(invalid_hll_err)); From 607ec36df04526f6a95aaaac701ed17387c7f204 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 18:57:24 +0200 Subject: [PATCH 090/320] RESP3: bool and null values in RESP -> human readable conversion. --- src/scripting.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 9ab1e7ab3..b76ff278a 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2155,6 +2155,8 @@ char *ldbRedisProtocolToHuman_Status(sds *o, char *reply); char *ldbRedisProtocolToHuman_MultiBulk(sds *o, char *reply); char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2171,6 +2173,8 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '*': p = ldbRedisProtocolToHuman_MultiBulk(o,reply); break; case '~': p = ldbRedisProtocolToHuman_Set(o,reply); break; case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; + case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; + case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; } return p; } @@ -2259,6 +2263,21 @@ char *ldbRedisProtocolToHuman_Map(sds *o, char *reply) { return p; } +char *ldbRedisProtocolToHuman_Null(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(null)",6); + return p+2; +} + +char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + if (reply[1] == 't') + *o = sdscatlen(*o,"#true",5); + else + *o = sdscatlen(*o,"#false",6); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From f3ac14411563128589448e17371efe7d8c0d8786 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:08:33 +0200 Subject: [PATCH 091/320] RESP3: double -> human readable conversion. --- src/scripting.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index b76ff278a..456210533 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2157,6 +2157,7 @@ char *ldbRedisProtocolToHuman_Set(sds *o, char *reply); char *ldbRedisProtocolToHuman_Map(sds *o, char *reply); char *ldbRedisProtocolToHuman_Null(sds *o, char *reply); char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply); +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply); /* Get Redis protocol from 'reply' and appends it in human readable form to * the passed SDS string 'o'. @@ -2175,6 +2176,7 @@ char *ldbRedisProtocolToHuman(sds *o, char *reply) { case '%': p = ldbRedisProtocolToHuman_Map(o,reply); break; case '_': p = ldbRedisProtocolToHuman_Null(o,reply); break; case '#': p = ldbRedisProtocolToHuman_Bool(o,reply); break; + case ',': p = ldbRedisProtocolToHuman_Double(o,reply); break; } return p; } @@ -2278,6 +2280,13 @@ char *ldbRedisProtocolToHuman_Bool(sds *o, char *reply) { return p+2; } +char *ldbRedisProtocolToHuman_Double(sds *o, char *reply) { + char *p = strchr(reply+1,'\r'); + *o = sdscatlen(*o,"(double) ",9); + *o = sdscatlen(*o,reply+1,p-reply-1); + return p+2; +} + /* Log a Redis reply as debugger output, in an human readable format. * If the resulting string is longer than 'len' plus a few more chars * used as prefix, it gets truncated. */ From 11678504f710021c3a6bea7c0568972d74e388f8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:20:30 +0200 Subject: [PATCH 092/320] RESP3: RESP3 double -> Lua conversion. --- src/scripting.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 456210533..7230f7438 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -45,6 +45,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype); char *redisProtocolToLuaType_Null(lua_State *lua, char *reply); char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf); +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); void ldbInit(void); @@ -139,6 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; } @@ -239,6 +241,27 @@ char *redisProtocolToLuaType_Bool(lua_State *lua, char *reply, int tf) { return p+2; } +char *redisProtocolToLuaType_Double(lua_State *lua, char *reply) { + char *p = strchr(reply+1,'\r'); + char buf[MAX_LONG_DOUBLE_CHARS+1]; + size_t len = p-reply-1; + double d; + + if (len <= MAX_LONG_DOUBLE_CHARS) { + memcpy(buf,reply+1,len); + buf[len] = '\0'; + d = strtod(buf,NULL); /* We expect a valid representation. */ + } else { + d = 0; + } + + lua_newtable(lua); + lua_pushstring(lua,"double"); + lua_pushnumber(lua,d); + lua_settable(lua,-3); + return p+2; +} + /* This function is used in order to push an error on the Lua stack in the * format used by redis.pcall to return errors, which is a lua table * with a single "err" field set to the error string. Note that this From eb464f737cb21f47d4a4ae89d8d7d84b46e4dc62 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2019 19:26:46 +0200 Subject: [PATCH 093/320] RESP3: Lua double -> RESP3 conversion. --- src/scripting.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 7230f7438..2153233f5 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -389,6 +389,17 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { } lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle double reply. */ + lua_pushstring(lua,"double"); + lua_gettable(lua,-2); + t = lua_type(lua,-1); + if (t == LUA_TNUMBER) { + addReplyDouble(c,lua_tonumber(lua,-1)); + lua_pop(lua,2); + return; + } + lua_pop(lua,1); /* Discard field name pushed before. */ + /* Handle map reply. */ lua_pushstring(lua,"map"); lua_gettable(lua,-2); From 238fd6546cbe58d031fb4121f495db37932a041a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:28:39 +0200 Subject: [PATCH 094/320] RESP3: Verbatim conversion test in the LOLWUT command. redis-cli is currently not able to handle it after going in RESP3 mode, because of hiredis limitations. --- src/lolwut.c | 3 ++- src/lolwut5.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index 19cbcf642..ba7e1069e 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -43,7 +43,8 @@ void lolwutUnstableCommand(client *c) { sds rendered = sdsnew("Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); } void lolwutCommand(client *c) { diff --git a/src/lolwut5.c b/src/lolwut5.c index 8408b378d..52a98c0d7 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -277,6 +277,7 @@ void lolwut5Command(client *c) { "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); - addReplyBulkSds(c,rendered); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); lwFreeCanvas(canvas); } From bbc1faa9d633db46a8064b2deb33f54d2097899b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:33:01 +0200 Subject: [PATCH 095/320] RESP3: Use verbatim in INFO output. --- src/server.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index fb308e5f8..7882b0d99 100644 --- a/src/server.c +++ b/src/server.c @@ -4391,7 +4391,9 @@ void infoCommand(client *c) { addReply(c,shared.syntaxerr); return; } - addReplyBulkSds(c, genRedisInfoString(section)); + sds info = genRedisInfoString(section); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } void monitorCommand(client *c) { From 90f06b04b009d8a47218a7f08b41cc9318103fb8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:46:11 +0200 Subject: [PATCH 096/320] RESP3: Use verbatim in DEBUG HTSTATS / HTSTATS-KEY. --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..93173f3b9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -638,7 +638,8 @@ NULL dictGetStats(buf,sizeof(buf),server.db[dbid].expires); stats = sdscat(stats,buf); - addReplyBulkSds(c,stats); + addReplyVerbatim(c,stats,sdslen(stats),"txt"); + sdsfree(stats); } else if (!strcasecmp(c->argv[1]->ptr,"htstats-key") && c->argc == 3) { robj *o; dict *ht = NULL; @@ -665,7 +666,7 @@ NULL } else { char buf[4096]; dictGetStats(buf,sizeof(buf),ht); - addReplyBulkCString(c,buf); + addReplyVerbatim(c,buf,strlen(buf),"txt"); } } else if (!strcasecmp(c->argv[1]->ptr,"change-repl-id") && c->argc == 2) { serverLog(LL_WARNING,"Changing replication IDs after receiving DEBUG change-repl-id"); From b98016ca055b55c841d63dc175f3ded2368389da Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:48:14 +0200 Subject: [PATCH 097/320] RESP3: Use verbatim in MEMORY subcommands. --- src/object.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index fb011d311..4534d7c06 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,13 +1440,15 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyBulkSds(c, info); + addReplyVerbatim(c,info,sdslen(info),"txt") + sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); #endif } else if (!strcasecmp(c->argv[1]->ptr,"doctor") && c->argc == 2) { sds report = getMemoryDoctorReport(); - addReplyBulkSds(c,report); + addReplyVerbatim(c,report,sdslen(report),"txt"); + sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { #if defined(USE_JEMALLOC) char tmp[32]; From 2c96a98ed8d552c3ec9f1f63f589747737d4f77c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:51:15 +0200 Subject: [PATCH 098/320] RESP3: Use verbatim in CLUSTER subcommands. --- src/cluster.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index a2615fdc0..1e7dcd50e 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4252,7 +4252,9 @@ NULL } } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { /* CLUSTER NODES */ - addReplyBulkSds(c,clusterGenNodesDescription(0)); + sds nodes = clusterGenNodesDescription(0); + addReplyVerbatim(c,nodes,sdslen(nodes),"txt"); + sdsfree(nodes); } else if (!strcasecmp(c->argv[1]->ptr,"myid") && c->argc == 2) { /* CLUSTER MYID */ addReplyBulkCBuffer(c,myself->name, CLUSTER_NAMELEN); @@ -4494,10 +4496,8 @@ NULL "cluster_stats_messages_received:%lld\r\n", tot_msg_received); /* Produce the reply protocol. */ - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); + addReplyVerbatim(c,info,sdslen(info),"txt"); + sdsfree(info); } else if (!strcasecmp(c->argv[1]->ptr,"saveconfig") && c->argc == 2) { int retval = clusterSaveConfig(1); From 90cf427a5ca6f92c52f09ba9bc9dddd66a00316f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:52:13 +0200 Subject: [PATCH 099/320] RESP3: Use verbatim in CLIENT LIST. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7555ca77d..a959d557a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1990,7 +1990,7 @@ NULL return; } sds o = getAllClientsInfoString(type); - addReplyBulkCBuffer(c,o,sdslen(o)); + addReplyVerbatim(c,o,sdslen(o),"txt"); sdsfree(o); } else if (!strcasecmp(c->argv[1]->ptr,"reply") && c->argc == 3) { /* CLIENT REPLY ON|OFF|SKIP */ From 747b4dd62f6061c735bef16b87225575e1c911ec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Sep 2019 18:53:22 +0200 Subject: [PATCH 100/320] RESP3: Use verbatim in LATENCY subcommands. --- src/latency.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/latency.c b/src/latency.c index 33aa1245b..b834da5c7 100644 --- a/src/latency.c +++ b/src/latency.c @@ -599,7 +599,7 @@ NULL event = dictGetKey(de); graph = latencyCommandGenSparkeline(event,ts); - addReplyBulkCString(c,graph); + addReplyVerbatim(c,graph,sdslen(graph),"txt"); sdsfree(graph); } else if (!strcasecmp(c->argv[1]->ptr,"latest") && c->argc == 2) { /* LATENCY LATEST */ @@ -608,7 +608,7 @@ NULL /* LATENCY DOCTOR */ sds report = createLatencyReport(); - addReplyBulkCBuffer(c,report,sdslen(report)); + addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"reset") && c->argc >= 2) { /* LATENCY RESET */ From dd077faf49d66019741805010b65918fa7fde24e Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:14:36 +0800 Subject: [PATCH 101/320] Fix bad handling of unexpected option while loading config "lua-replicate-commands". --- src/config.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config.c b/src/config.c index a72df2e78..ae4d4a0e6 100644 --- a/src/config.c +++ b/src/config.c @@ -672,6 +672,9 @@ void loadServerConfigFromString(char *config) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { server.lua_always_replicate_commands = yesnotoi(argv[1]); + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { From c6454647838a0fd56ed807de9ddf45bdeb5cfd90 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Fri, 20 Sep 2019 08:37:23 +0800 Subject: [PATCH 102/320] RESP3: Fix function redisProtocolToLuaType about RESP3->Lua bools. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 2153233f5..fa896de21 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -139,7 +139,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '%': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; - case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); + case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; case ',': p = redisProtocolToLuaType_Double(lua,reply); } return p; From 4429a40317c386e756f5f539f79c56502207f1d3 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Fri, 20 Sep 2019 01:11:20 -0700 Subject: [PATCH 103/320] Fix compilation error --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index 4534d7c06..697429b84 100644 --- a/src/object.c +++ b/src/object.c @@ -1440,7 +1440,7 @@ NULL #if defined(USE_JEMALLOC) sds info = sdsempty(); je_malloc_stats_print(inputCatSds, &info, NULL); - addReplyVerbatim(c,info,sdslen(info),"txt") + addReplyVerbatim(c,info,sdslen(info),"txt"); sdsfree(info); #else addReplyBulkCString(c,"Stats not supported for the current allocator"); From 5cb591ca40ca424779dd97a47dece41831fce97a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:18:59 +0200 Subject: [PATCH 104/320] Add useless break for uniformity / future protection. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index fa896de21..9ac8af2e7 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -140,7 +140,7 @@ char *redisProtocolToLuaType(lua_State *lua, char* reply) { case '~': p = redisProtocolToLuaType_Aggregate(lua,reply,*p); break; case '_': p = redisProtocolToLuaType_Null(lua,reply); break; case '#': p = redisProtocolToLuaType_Bool(lua,reply,p[1]); break; - case ',': p = redisProtocolToLuaType_Double(lua,reply); + case ',': p = redisProtocolToLuaType_Double(lua,reply); break; } return p; } From e7444fde2d1776f408a6f006c21ee97f5afcd9cd Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:44:32 +0200 Subject: [PATCH 105/320] Remove redundant statement in config.c. Thanks to @guybe7 for spotting the error in the original PR I merged. --- src/config.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index ae4d4a0e6..fe002bac2 100644 --- a/src/config.c +++ b/src/config.c @@ -671,8 +671,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) == -1) { + if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) + == -1) + { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && From 31a20f0fbb6cdd59fa404bd22dcd62806e88f50e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 11:46:35 +0200 Subject: [PATCH 106/320] Make config.c always_replicate_commands more uniform. Better if it resembles the other similar code paths. --- src/config.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index fe002bac2..d37e3c566 100644 --- a/src/config.c +++ b/src/config.c @@ -671,10 +671,10 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - if ((server.lua_always_replicate_commands = yesnotoi(argv[1])) - == -1) - { - err = "argument must be 'yes' or 'no'"; goto loaderr; + server.lua_always_replicate_commands = yesnotoi(argv[1]); + if (server.lua_always_replicate_commands == -1) { + err = "argument must be 'yes' or 'no'"; + goto loaderr; } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) From d48e956b6f838a47949b5e8939e0424c0366ebde Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 20 Sep 2019 11:01:36 +0100 Subject: [PATCH 107/320] Adding AnonHugePages case + comments --- src/zmalloc.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 48f4b1e99..764738636 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -395,15 +395,24 @@ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { return bytes; } #else +/* Get sum of the specified field from libproc api call. + * As there are per page value basis we need to convert + * them accordingly. + * + * Note that AnonHugePages is a no-op as THP feature + * is not supported in this platform + */ size_t zmalloc_get_smap_bytes_by_field(char *field, long pid) { #if defined(__APPLE__) struct proc_regioninfo pri; if (proc_pidinfo(pid, PROC_PIDREGIONINFO, 0, &pri, PROC_PIDREGIONINFO_SIZE) == PROC_PIDREGIONINFO_SIZE) { - if (!strcmp(field, "Private_Dirty")) { + if (!strcmp(field, "Private_Dirty:")) { return (size_t)pri.pri_pages_dirtied * 4096; - } else if (!strcmp(field, "Rss")) { + } else if (!strcmp(field, "Rss:")) { return (size_t)pri.pri_pages_resident * 4096; + } else if (!strcmp(field, "AnonHugePages:")) { + return 0; } } return 0; From a84c87cc95a347b4b3a2caa9c7ca1aa0415e21c0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Sep 2019 20:06:47 +0200 Subject: [PATCH 108/320] hiredis updated to master version. --- deps/hiredis/.gitignore | 1 + deps/hiredis/.travis.yml | 74 ++- deps/hiredis/CHANGELOG.md | 15 +- deps/hiredis/CMakeLists.txt | 90 ++++ deps/hiredis/Makefile | 106 ++++- deps/hiredis/README.md | 3 +- deps/hiredis/adapters/libevent.h | 112 ++++- deps/hiredis/appveyor.yml | 7 +- deps/hiredis/async.c | 176 +++++--- deps/hiredis/async.h | 8 + deps/hiredis/async_private.h | 72 +++ deps/hiredis/examples/CMakeLists.txt | 46 ++ deps/hiredis/examples/example-libevent-ssl.c | 73 +++ deps/hiredis/examples/example-libevent.c | 15 +- deps/hiredis/examples/example-ssl.c | 97 ++++ deps/hiredis/examples/example.c | 17 +- deps/hiredis/hiredis.c | 253 ++++++----- deps/hiredis/hiredis.h | 103 ++++- deps/hiredis/hiredis.pc.in | 11 + deps/hiredis/hiredis_ssl.h | 53 +++ deps/hiredis/hiredis_ssl.pc.in | 12 + deps/hiredis/net.c | 122 +++-- deps/hiredis/net.h | 4 + deps/hiredis/read.c | 28 +- deps/hiredis/read.h | 5 +- deps/hiredis/sds.c | 2 +- deps/hiredis/sds.h | 31 +- deps/hiredis/sockcompat.c | 248 ++++++++++ deps/hiredis/sockcompat.h | 91 ++++ deps/hiredis/ssl.c | 448 +++++++++++++++++++ deps/hiredis/test.c | 93 +++- deps/hiredis/test.sh | 70 +++ deps/hiredis/win32.h | 18 +- 33 files changed, 2178 insertions(+), 326 deletions(-) create mode 100644 deps/hiredis/CMakeLists.txt create mode 100644 deps/hiredis/async_private.h create mode 100644 deps/hiredis/examples/CMakeLists.txt create mode 100644 deps/hiredis/examples/example-libevent-ssl.c create mode 100644 deps/hiredis/examples/example-ssl.c create mode 100644 deps/hiredis/hiredis.pc.in create mode 100644 deps/hiredis/hiredis_ssl.h create mode 100644 deps/hiredis/hiredis_ssl.pc.in create mode 100644 deps/hiredis/sockcompat.c create mode 100644 deps/hiredis/sockcompat.h create mode 100644 deps/hiredis/ssl.c create mode 100755 deps/hiredis/test.sh diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index c44b5c537..8e50b5434 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index faf2ce684..dd8e0e73d 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -26,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" + +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index a7fe3ac11..d1d37e515 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -12,6 +12,16 @@ compare to other values, casting might be necessary or can be removed, if casting was applied before. +### 0.x.x (unreleased) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + ### 0.14.0 (2018-09-25) * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) @@ -50,8 +60,9 @@ * Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 * Fix warnings, when compiled with -Wshadow * Make hiredis compile in Cygwin on Windows, now CI-tested - -**BREAKING CHANGES**: +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt new file mode 100644 index 000000000..9e78894f3 --- /dev/null +++ b/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 06ca99468..25ac15464 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -39,7 +45,7 @@ export REDIS_TEST_CONFIG CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) REAL_LDFLAGS=$(LDFLAGS) @@ -49,12 +55,30 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) @@ -66,40 +90,61 @@ ifeq ($(uname_S),Darwin) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -116,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -133,32 +178,33 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c INSTALL?= cp -pPR @@ -175,6 +221,20 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 01223ea59..c0b432f07 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 7d2bef180..a4952776c 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_free(e->rev); - event_free(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml index 819efbd58..5b43fdbeb 100644 --- a/deps/hiredis/appveyor.yml +++ b/deps/hiredis/appveyor.yml @@ -5,8 +5,9 @@ environment: CC: gcc - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 clone_depth: 1 @@ -20,4 +21,4 @@ install: build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -344,9 +330,15 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -358,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -408,7 +403,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, assert(reply->element[2]->type == REDIS_REPLY_INTEGER); /* Unset subscribed flag only when no pipelined pending subscribe. */ - if (reply->element[2]->integer == 0 + if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; @@ -524,6 +519,18 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { } } +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + /* This function should be called when the socket is readable. * It processes all replies that can be read and executes their callbacks. */ @@ -539,28 +546,13 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } + c->funcs->async_read(ac); } -void redisAsyncHandleWrite(redisAsyncContext *ac) { +void redisAsyncWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); int done = 0; - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { @@ -575,6 +567,51 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { } } +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + c->funcs->async_write(ac); +} + +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); +} + /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ static const char *nextArgument(const char *start, const char **str, size_t *len) { @@ -714,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 740555c24..4f6b3b783 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -57,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -81,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -106,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -113,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h new file mode 100644 index 000000000..d0133ae18 --- /dev/null +++ b/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 000000000..dd3a313ac --- /dev/null +++ b/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 000000000..1021113b9 --- /dev/null +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index d333c22b7..1fe71ae4e 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c new file mode 100644 index 000000000..81f4648c6 --- /dev/null +++ b/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 4d494c55a..0e93fc8b3 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 0947d1ed7..282595bdb 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,10 +41,20 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); @@ -112,19 +121,31 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len if (r == NULL) return NULL; - buf = malloc(len+1); - if (buf == NULL) { - freeReplyObject(r); - return NULL; - } - assert(task->type == REDIS_REPLY_ERROR || task->type == REDIS_REPLY_STATUS || - task->type == REDIS_REPLY_STRING); + task->type == REDIS_REPLY_STRING || + task->type == REDIS_REPLY_VERB); /* Copy string value */ - memcpy(buf,str,len); - buf[len] = '\0'; + if (task->type == REDIS_REPLY_VERB) { + buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(r->vtype,buf,3); + r->vtype[3] = '\0'; + memcpy(buf+4,str,len-4); + buf[len-4] = '\0'; + } else { + buf = malloc(len+1); + if (buf == NULL) { + freeReplyObject(r); + return NULL; + } + memcpy(buf,str,len); + buf[len] = '\0'; + } r->str = buf; r->len = len; @@ -138,7 +159,7 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; r = createReplyObject(task->type); @@ -649,29 +670,30 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); + redisNetClose(c); sdsfree(c->obuf); redisReaderFree(c->reader); @@ -680,12 +702,16 @@ void redisFree(redisContext *c) { free(c->unix_sock.path); free(c->timeout); free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -694,10 +720,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -718,112 +747,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -853,22 +877,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); + return REDIS_ERR; } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -883,21 +900,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 47d7982e9..69dc39c5e 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -35,7 +35,11 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ @@ -74,6 +78,12 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -92,6 +102,8 @@ typedef struct redisReply { size_t len; /* Length of string */ char *str; /* Used for REDIS_REPLY_ERROR, REDIS_REPLY_STRING and REDIS_REPLY_DOUBLE (in additionl to dval). */ + char vtype[4]; /* Used for REDIS_REPLY_VERB, contains the null + terminated 3 character content type, such as "txt". */ size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ } redisReply; @@ -111,14 +123,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -139,8 +230,12 @@ typedef struct redisContext { /* For non-blocking connect */ struct sockadr *saddr; size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -151,7 +246,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -167,7 +262,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in new file mode 100644 index 000000000..140b040f1 --- /dev/null +++ b/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h new file mode 100644 index 000000000..f844f9548 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 000000000..588a978a5 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index a4b3abc6d..e5f40b0a4 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -34,36 +34,64 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; @@ -79,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -101,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -108,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -119,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); + return REDIS_ERR; + } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,12 +249,12 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -230,7 +267,7 @@ static int redisContextWaitReady(redisContext *c, long msec) { } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -277,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -291,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -360,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -401,16 +446,14 @@ addrretry: } /* For repeat connection */ - if (c->saddr) { - free(c->saddr); - } + free(c->saddr); c->saddr = malloc(p->ai_addrlen); memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; } else if (errno == EINPROGRESS) { if (blocking) { @@ -424,7 +467,7 @@ addrretry: if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { @@ -471,8 +514,9 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; if (redisCreateSocket(c,AF_UNIX) < 0) @@ -499,9 +543,11 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_UNIX; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -516,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a11594e68..a4393c06b 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -37,6 +37,10 @@ #include "hiredis.h" +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); + int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index cc0f3cc72..b9853ea9a 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -31,10 +31,10 @@ #include "fmacros.h" #include -#include #include #ifndef _MSC_VER #include +#include #endif #include #include @@ -44,6 +44,7 @@ #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -294,9 +295,9 @@ static int processLineItem(redisReader *r) { buf[len] = '\0'; if (strcasecmp(buf,",inf") == 0) { - d = 1.0/0.0; /* Positive infinite. */ + d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -1.0/0.0; /* Nevative infinite. */ + d = -INFINITY; /* Nevative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -379,10 +380,18 @@ static int processBulkItem(redisReader *r) { /* Only continue when the buffer contains the entire bulk item. */ bytelen += len+2; /* include \r\n */ if (r->pos+bytelen <= r->len) { + if ((cur->type == REDIS_REPLY_VERB && len < 4) || + (cur->type == REDIS_REPLY_VERB && s[5] != ':')) + { + __redisReaderSetError(r,REDIS_ERR_PROTOCOL, + "Verbatim string 4 bytes of content type are " + "missing or incorrectly encoded."); + return REDIS_ERR; + } if (r->fn && r->fn->createString) obj = r->fn->createString(cur,s+2,len); else - obj = (void*)REDIS_REPLY_STRING; + obj = (void*)(long)cur->type; success = 1; } } @@ -430,7 +439,7 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || elements > INT_MAX) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -523,6 +532,9 @@ static int processItem(redisReader *r) { case '#': cur->type = REDIS_REPLY_BOOL; break; + case '=': + cur->type = REDIS_REPLY_VERB; + break; default: __redisReaderSetErrorProtocolByte(r,*p); return REDIS_ERR; @@ -543,6 +555,7 @@ static int processItem(redisReader *r) { case REDIS_REPLY_BOOL: return processLineItem(r); case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: return processBulkItem(r); case REDIS_REPLY_ARRAY: case REDIS_REPLY_MAP: @@ -657,8 +670,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index f3d075843..58105312a 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -55,12 +56,12 @@ #define REDIS_REPLY_ERROR 6 #define REDIS_REPLY_DOUBLE 7 #define REDIS_REPLY_BOOL 8 -#define REDIS_REPLY_VERB 9 #define REDIS_REPLY_MAP 9 #define REDIS_REPLY_SET 10 #define REDIS_REPLY_ATTR 11 #define REDIS_REPLY_PUSH 12 #define REDIS_REPLY_BIGNUM 13 +#define REDIS_REPLY_VERB 14 #define REDIS_READER_MAX_BUF (1024*16) /* Default max unused reader buffer. */ @@ -79,7 +80,7 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 44777b10c..6cf75841c 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1035,7 +1035,7 @@ sds *sdssplitargs(const char *line, int *argc) { s_free(vector); return NULL; } - + vector = new_vector; vector[*argc] = current; (*argc)++; diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 13be75a9f..3f9a96457 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -132,20 +135,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -156,21 +159,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -200,16 +203,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c new file mode 100644 index 000000000..4cc2f414f --- /dev/null +++ b/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h new file mode 100644 index 000000000..56006c163 --- /dev/null +++ b/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c new file mode 100644 index 000000000..78ab9e43e --- /dev/null +++ b/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 79cff4308..8668e1856 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -13,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -33,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -93,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -121,9 +149,21 @@ static redisContext *do_connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -360,7 +400,8 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); - test("Set error when array > INT_MAX: "); +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); ret = redisReaderGetReply(reader,&reply); @@ -369,7 +410,6 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); -#if LLONG_MAX > SIZE_MAX test("Set error when bulk > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); @@ -434,22 +474,23 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; struct addrinfo hints = {.ai_family = AF_INET}; struct addrinfo *ai_tmp = NULL; - const char *bad_domain = "idontexist.com"; - int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp); + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); if (rv != 0) { // Address does *not* exist test("Returns error when host cannot be resolved: "); // First see if this domain name *actually* resolves to NXDOMAIN - c = redisConnect("dontexist.com", 6379); + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); test_cond( c->err == REDIS_ERR_OTHER && (strcmp(c->errstr, "Name or service not known") == 0 || - strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || @@ -574,7 +615,8 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -583,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -591,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -894,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -922,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh new file mode 100755 index 000000000..2cab9e6fb --- /dev/null +++ b/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ From bd32b8b2a8bc743187bf6fc98de6163660268afe Mon Sep 17 00:00:00 2001 From: valentino Date: Sat, 21 Sep 2019 20:58:57 +0300 Subject: [PATCH 109/320] DISCARD should not fail during OOM discard command should not fail during OOM, otherwise client MULTI state will not be cleared. --- src/server.c | 2 +- tests/unit/multi.tcl | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..3737c3352 100644 --- a/src/server.c +++ b/src/server.c @@ -3411,7 +3411,7 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand))) { + (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 6655bf62c..9fcef71d6 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -306,4 +306,18 @@ start_server {tags {"multi"}} { } close_replication_stream $repl } + + test {DISCARD should not fail during OOM} { + set rd [redis_deferring_client] + $rd config set maxmemory 1 + assert {[$rd read] eq {OK}} + r multi + catch {r set x 1} e + assert_match {OOM*} $e + r discard + $rd config set maxmemory 0 + assert {[$rd read] eq {OK}} + $rd close + r ping + } {PONG} } From 8ceffc04558648c7a3758f46dab51debe2bd3d0a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:39:42 +0200 Subject: [PATCH 110/320] RESP3: implementation of verbatim output with TTY target. --- src/redis-cli.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index db53fc7d8..d08dfeb5f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -819,10 +819,17 @@ static sds cliFormatReplyTTY(redisReply *r, char *prefix) { out = sdscatprintf(out,"(double) %s\n",r->str); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: /* If you are producing output for the standard output we want - * a more interesting output with quoted characters and so forth */ - out = sdscatrepr(out,r->str,r->len); - out = sdscat(out,"\n"); + * a more interesting output with quoted characters and so forth, + * unless it's a verbatim string type. */ + if (r->type == REDIS_REPLY_STRING) { + out = sdscatrepr(out,r->str,r->len); + out = sdscat(out,"\n"); + } else { + out = sdscatlen(out,r->str,r->len); + out = sdscat(out,"\n"); + } break; case REDIS_REPLY_NIL: out = sdscat(out,"(nil)\n"); From d097eebf0694987d48d59fbbd2da7eeff15c0a03 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:41:04 +0200 Subject: [PATCH 111/320] hiredis udpated (RESP3 WIP). --- deps/hiredis/hiredis.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 282595bdb..abd94c01d 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -128,15 +128,16 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len /* Copy string value */ if (task->type == REDIS_REPLY_VERB) { - buf = malloc(len+4+1); /* Skip 4 bytes of verbatim type header. */ + buf = malloc(len-4+1); /* Skip 4 bytes of verbatim type header. */ if (buf == NULL) { freeReplyObject(r); return NULL; } - memcpy(r->vtype,buf,3); + memcpy(r->vtype,str,3); r->vtype[3] = '\0'; - memcpy(buf+4,str,len-4); + memcpy(buf,str+4,len-4); buf[len-4] = '\0'; + r->len = len-4; } else { buf = malloc(len+1); if (buf == NULL) { @@ -145,9 +146,9 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len } memcpy(buf,str,len); buf[len] = '\0'; + r->len = len; } r->str = buf; - r->len = len; if (task->parent) { parent = task->parent->obj; From cd446eb161b02db9a9affea8ba916c1c1ad6fa76 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:42:57 +0200 Subject: [PATCH 112/320] RESP3: varbatim handling for other redis-cli outputs. --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index d08dfeb5f..9a5f81723 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -968,6 +968,7 @@ static sds cliFormatReplyRaw(redisReply *r) { break; case REDIS_REPLY_STATUS: case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: if (r->type == REDIS_REPLY_STATUS && config.eval_ldb) { /* The Lua debugger replies with arrays of simple (status) * strings. We colorize the output for more fun if this @@ -1021,6 +1022,7 @@ static sds cliFormatReplyCSV(redisReply *r) { out = sdscatprintf(out,"%lld",r->integer); break; case REDIS_REPLY_STRING: + case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: From 24a78adf1063653adc80187139e01233fdb3d5b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:47:36 +0200 Subject: [PATCH 113/320] redis-cli: AUTH can now have 3 arguments as well. --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 9a5f81723..38a0b0b7f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1222,7 +1222,8 @@ static int cliSendCommand(int argc, char **argv, long repeat) { if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) { config.dbnum = atoi(argv[1]); cliRefreshPrompt(); - } else if (!strcasecmp(command,"auth") && argc == 2) { + } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) + { cliSelect(); } } From ddb735f03a6b43700a8c870254fcf4e00a6a7da6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 17:55:05 +0200 Subject: [PATCH 114/320] redis-cli: ability to start a session in RESP3 mode. --- src/redis-cli.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 38a0b0b7f..47bd11a8a 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -230,6 +230,7 @@ static struct config { int verbose; clusterManagerCommand cluster_manager_command; int no_auth_warning; + int resp3; } config; /* User preferences. */ @@ -751,6 +752,21 @@ static int cliSelect(void) { return REDIS_ERR; } +/* Select RESP3 mode if redis-cli was started with the -3 option. */ +static int cliSwitchProto(void) { + redisReply *reply; + if (config.resp3 == 0) return REDIS_OK; + + reply = redisCommand(context,"HELLO 3"); + if (reply != NULL) { + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR; + freeReplyObject(reply); + return result; + } + return REDIS_ERR; +} + /* Connect to the server. It is possible to pass certain flags to the function: * CC_FORCE: The connection is performed even if there is already * a connected socket. @@ -788,11 +804,13 @@ static int cliConnect(int flags) { * errors. */ anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); - /* Do AUTH and select the right DB. */ + /* Do AUTH, select the right DB, switch to RESP3 if needed. */ if (cliAuth() != REDIS_OK) return REDIS_ERR; if (cliSelect() != REDIS_OK) return REDIS_ERR; + if (cliSwitchProto() != REDIS_OK) + return REDIS_ERR; } return REDIS_OK; } @@ -1449,6 +1467,8 @@ static int parseOptions(int argc, char **argv) { printf("redis-cli %s\n", version); sdsfree(version); exit(0); + } else if (!strcmp(argv[i],"-3")) { + config.resp3 = 1; } else if (CLUSTER_MANAGER_MODE() && argv[i][0] != '-') { if (config.cluster_manager_command.argc == 0) { int j = i + 1; @@ -1529,6 +1549,7 @@ static void usage(void) { " -i When -r is used, waits seconds per command.\n" " It is possible to specify sub-second times like -i 0.1.\n" " -n Database number.\n" +" -3 Start session in RESP3 protocol mode.\n" " -x Read last argument from STDIN.\n" " -d Multi-bulk delimiter in for raw formatting (default: \\n).\n" " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" @@ -1543,7 +1564,9 @@ static void usage(void) { " --csv is specified, or if you redirect the output to a non\n" " TTY, it samples the latency for 1 second (you can use\n" " -i to change the interval), then produces a single output\n" -" and exits.\n" +" and exits.\n",version); + + fprintf(stderr, " --latency-history Like --latency but tracking latency changes over time.\n" " Default time interval is 15 sec. Change it using -i.\n" " --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n" @@ -1578,7 +1601,7 @@ static void usage(void) { " --help Output this help and exit.\n" " --version Output version and exit.\n" "\n", - version, REDIS_CLI_DEFAULT_PIPE_TIMEOUT); + REDIS_CLI_DEFAULT_PIPE_TIMEOUT); /* Using another fprintf call to avoid -Woverlength-strings compile warning */ fprintf(stderr, "Cluster Manager Commands:\n" From c7481c6af1441f272fa4f8b6b9b559ae5aa64b73 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:36:06 +0200 Subject: [PATCH 115/320] redis-cli: CSV and RAW target for more RESP3 types. --- src/redis-cli.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 47bd11a8a..7374526e5 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1006,9 +1006,15 @@ static sds cliFormatReplyRaw(redisReply *r) { out = sdscatlen(out,r->str,r->len); } break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "(true)" : "(false)"); + break; case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_ARRAY: for (i = 0; i < r->elements; i++) { if (i > 0) out = sdscat(out,config.mb_delim); @@ -1017,6 +1023,19 @@ static sds cliFormatReplyRaw(redisReply *r) { sdsfree(tmp); } break; + case REDIS_REPLY_MAP: + for (i = 0; i < r->elements; i += 2) { + if (i > 0) out = sdscat(out,config.mb_delim); + tmp = cliFormatReplyRaw(r->element[i]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + + out = sdscatlen(out," ",1); + tmp = cliFormatReplyRaw(r->element[i+1]); + out = sdscatlen(out,tmp,sdslen(tmp)); + sdsfree(tmp); + } + break; default: fprintf(stderr,"Unknown reply type: %d\n", r->type); exit(1); @@ -1039,14 +1058,21 @@ static sds cliFormatReplyCSV(redisReply *r) { case REDIS_REPLY_INTEGER: out = sdscatprintf(out,"%lld",r->integer); break; + case REDIS_REPLY_DOUBLE: + out = sdscatprintf(out,"%s",r->str); + break; case REDIS_REPLY_STRING: case REDIS_REPLY_VERB: out = sdscatrepr(out,r->str,r->len); break; case REDIS_REPLY_NIL: - out = sdscat(out,"NIL"); + out = sdscat(out,"NULL"); + break; + case REDIS_REPLY_BOOL: + out = sdscat(out,r->integer ? "true" : "false"); break; case REDIS_REPLY_ARRAY: + case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */ for (i = 0; i < r->elements; i++) { sds tmp = cliFormatReplyCSV(r->element[i]); out = sdscatlen(out,tmp,sdslen(tmp)); From fddbae80faf422562ce4d5db305de4a688090490 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Sep 2019 19:57:13 +0200 Subject: [PATCH 116/320] redis-cli: support for ACL style user/pass AUTH. --- src/redis-cli.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7374526e5..c183155cb 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -218,6 +218,7 @@ static struct config { int hotkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; + char *user; int output; /* output mode, see OUTPUT_* defines */ sds mb_delim; char prompt[128]; @@ -729,8 +730,13 @@ static int cliAuth(void) { redisReply *reply; if (config.auth == NULL) return REDIS_OK; - reply = redisCommand(context,"AUTH %s",config.auth); + if (config.user == NULL) + reply = redisCommand(context,"AUTH %s",config.auth); + else + reply = redisCommand(context,"AUTH %s %s",config.user,config.auth); if (reply != NULL) { + if (reply->type == REDIS_REPLY_ERROR) + fprintf(stderr,"Warning: AUTH failed\n"); freeReplyObject(reply); return REDIS_OK; } @@ -1350,8 +1356,12 @@ static int parseOptions(int argc, char **argv) { config.dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; - } else if (!strcmp(argv[i],"-a") && !lastarg) { + } else if ((!strcmp(argv[i],"-a") || !strcmp(argv[i],"--pass")) + && !lastarg) + { config.auth = argv[++i]; + } else if (!strcmp(argv[i],"--user") && !lastarg) { + config.user = argv[++i]; } else if (!strcmp(argv[i],"-u") && !lastarg) { parseRedisUri(argv[++i]); } else if (!strcmp(argv[i],"--raw")) { @@ -1570,6 +1580,8 @@ static void usage(void) { " You can also use the " REDIS_CLI_AUTH_ENV " environment\n" " variable to pass this password more safely\n" " (if both are used, this argument takes predecence).\n" +" -user Used to send ACL style 'AUTH username pass'. Needs -a.\n" +" -pass Alias of -a for consistency with the new --user option.\n" " -u Server URI.\n" " -r Execute specified command N times.\n" " -i When -r is used, waits seconds per command.\n" @@ -2409,7 +2421,12 @@ static int clusterManagerNodeConnect(clusterManagerNode *node) { * errors. */ anetKeepAlive(NULL, node->context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); if (config.auth) { - redisReply *reply = redisCommand(node->context,"AUTH %s",config.auth); + redisReply *reply; + if (config.user == NULL) + reply = redisCommand(node->context,"AUTH %s", config.auth); + else + reply = redisCommand(node->context,"AUTH %s %s", + config.user,config.auth); int ok = clusterManagerCheckRedisReply(node, reply, NULL); if (reply != NULL) freeReplyObject(reply); if (!ok) return 0; @@ -7737,6 +7754,7 @@ int main(int argc, char **argv) { config.hotkeys = 0; config.stdinarg = 0; config.auth = NULL; + config.user = NULL; config.eval = NULL; config.eval_ldb = 0; config.eval_ldb_end = 0; From 26452e0d85c9b2e2c6a523b16109ba0087c1321b Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Mon, 23 Sep 2019 23:45:31 +0100 Subject: [PATCH 117/320] [fix] un-refactor the code. [perf] replyWithStatus now makes usage of addReplyProto --- src/module.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/module.c b/src/module.c index ad7e6864c..067a69928 100644 --- a/src/module.c +++ b/src/module.c @@ -1120,6 +1120,19 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { return REDISMODULE_OK; } +/* Reply with an error or simple string (status message). Used to implement + * ReplyWithSimpleString() and ReplyWithError(). + * The function always returns REDISMODULE_OK. */ +int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + const size_t len = strlen(msg); + addReplyProto(c,"-",1); + addReplyProto(c,msg,len); + addReplyProto(c,"\r\n",2); + return REDISMODULE_OK; +} + /* Reply with the error 'err'. * * Note that 'err' must contain all the error, including @@ -1135,13 +1148,7 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { * The function always returns REDISMODULE_OK. */ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(err); - addReplyProto(c,"-",1); - addReplyProto(c,err,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,err,"-"); } /* Reply with a simple string (+... \r\n in RESP protocol). This replies @@ -1150,13 +1157,7 @@ int RM_ReplyWithError(RedisModuleCtx *ctx, const char *err) { * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithSimpleString(RedisModuleCtx *ctx, const char *msg) { - client *c = moduleGetReplyClient(ctx); - if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"+",1); - addReplyProto(c,msg,len); - addReplyProto(c,"\r\n",2); - return REDISMODULE_OK; + return replyWithStatus(ctx,msg,"+"); } /* Reply with an array type of 'len' elements. However 'len' other calls From c9a39e9bc5bc40757b50408bda2a95c319829357 Mon Sep 17 00:00:00 2001 From: "Mike A. Owens" Date: Mon, 23 Sep 2019 19:24:09 -0400 Subject: [PATCH 118/320] Seed SipHash with 128-bit key SipHash expects a 128-bit key, and we were indeed generating 128-bits, but restricting them to hex characters 0-9a-f, effectively giving us only 4 bits-per-byte of key material, and 64 bits overall. Now, we skip the hex conversion and supply 128 bits of unfiltered random data. --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..fc9809b1c 100644 --- a/src/server.c +++ b/src/server.c @@ -4787,9 +4787,9 @@ int main(int argc, char **argv) { srand(time(NULL)^getpid()); gettimeofday(&tv,NULL); - char hashseed[16]; - getRandomHexChars(hashseed,sizeof(hashseed)); - dictSetHashFunctionSeed((uint8_t*)hashseed); + uint8_t hashseed[16]; + getRandomBytes(hashseed,sizeof(hashseed)); + dictSetHashFunctionSeed(hashseed); server.sentinel_mode = checkForSentinelMode(argc,argv); initServerConfig(); ACLInit(); /* The ACL subsystem must be initialized ASAP because the From 53e3b994f07cfbcd4eee3dec76c7b180c75c709f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 17:45:05 +0200 Subject: [PATCH 119/320] ACL: fix ##6408, default user state affecting all the connections. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 7882b0d99..2bdfd1640 100644 --- a/src/server.c +++ b/src/server.c @@ -3341,9 +3341,10 @@ int processCommand(client *c) { /* Check if the user is authenticated. This check is skipped in case * the default user is flagged as "nopass" and is active. */ - int auth_required = !(DefaultUser->flags & USER_FLAG_NOPASS) && + int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || + DefaultUser->flags & USER_FLAG_DISABLED) && !c->authenticated; - if (auth_required || DefaultUser->flags & USER_FLAG_DISABLED) { + if (auth_required) { /* AUTH and HELLO are valid even in non authenticated state. */ if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { flagTransaction(c); From 73eac241659e7dfdf81f41b7242ec1c83a0b966f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Sep 2019 18:08:11 +0200 Subject: [PATCH 120/320] Modify #6401 changes to fit 80 cols. --- src/server.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 42dc8f1e4..51106abe0 100644 --- a/src/server.c +++ b/src/server.c @@ -3412,7 +3412,10 @@ int processCommand(client *c) { * is in MULTI/EXEC context? Error. */ if (out_of_memory && (c->cmd->flags & CMD_DENYOOM || - (c->flags & CLIENT_MULTI && c->cmd->proc != execCommand && c->cmd->proc != discardCommand))) { + (c->flags & CLIENT_MULTI && + c->cmd->proc != execCommand && + c->cmd->proc != discardCommand))) + { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; From 02ddcc0c78e7537cf4cabb4fc593771f901b01db Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 25 Sep 2019 17:28:42 +0100 Subject: [PATCH 121/320] [fix] fixed the un-refactor bug. --- src/module.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 067a69928..d02e2708c 100644 --- a/src/module.c +++ b/src/module.c @@ -1126,9 +1126,10 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t len = strlen(msg); - addReplyProto(c,"-",1); - addReplyProto(c,msg,len); + const size_t msgLen = strlen(msg); + const size_t prefixLen = strlen(prefix); + addReplyProto(c,prefix,prefixLen); + addReplyProto(c,msg,msgLen); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From cf2dc0779133c8be10c73d22345521c3e9204a75 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 12:18:39 +0200 Subject: [PATCH 122/320] INFO: more info about loaded modules. Related to #6024. --- src/module.c | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 71fdd6b74..b2bddd316 100644 --- a/src/module.c +++ b/src/module.c @@ -52,7 +52,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ - int options; /* Moduile options and capabilities. */ + int options; /* Module options and capabilities. */ }; typedef struct RedisModule RedisModule; @@ -5363,9 +5363,36 @@ void addReplyLoadedModules(client *c) { dictReleaseIterator(di); } +/* Helper for genModulesInfoString(): given a list of modules, return + * am SDS string in the form "[modulename|modulename2|...]" */ +sds genModulesInfoStringRenderModulesList(list *l) { + listIter li; + listNode *ln; + listRewind(l,&li); + sds output = sdsnew("["); + while((ln = listNext(&li))) { + RedisModule *module = ln->value; + output = sdscat(output,module->name); + } + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + +/* Helper for genModulesInfoString(): render module options as an SDS string. */ +sds genModulesInfoStringRenderModuleOptions(struct RedisModule *module) { + sds output = sdsnew("["); + if (module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) + output = sdscat(output,"handle-io-errors|"); + output = sdstrim(output,"|"); + output = sdscat(output,"]"); + return output; +} + + /* Helper function for the INFO command: adds loaded modules as to info's * output. - * + * * After the call, the passed sds info string is no longer valid and all the * references must be substituted with the new pointer returned by the call. */ sds genModulesInfoString(sds info) { @@ -5376,7 +5403,17 @@ sds genModulesInfoString(sds info) { sds name = dictGetKey(de); struct RedisModule *module = dictGetVal(de); - info = sdscatprintf(info, "module:name=%s,ver=%d\r\n", name, module->ver); + sds usedby = genModulesInfoStringRenderModulesList(module->usedby); + sds using = genModulesInfoStringRenderModulesList(module->using); + sds options = genModulesInfoStringRenderModuleOptions(module); + info = sdscatprintf(info, + "module:name=%s,ver=%d,api=%d,filters=%d," + "usedby=%s,using=%s,options=%s\r\n", + name, module->ver, module->apiver, + (int)listLength(module->filters), usedby, using, options); + sdsfree(usedby); + sdsfree(using); + sdsfree(options); } dictReleaseIterator(di); return info; From 2add4524c533ec1fdafccf28300e56d0ea6e16b7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 26 Sep 2019 15:16:34 +0300 Subject: [PATCH 123/320] Fix lastbgsave_status, when new child signal handler get intended kill And add a test for that. --- src/server.c | 9 ++++++++- tests/integration/rdb.tcl | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 31418e01b..c0e59c86c 100644 --- a/src/server.c +++ b/src/server.c @@ -1910,6 +1910,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); + /* sigKillChildHandler catches the signal and calls exit(), but we + * must make sure not to flag lastbgsave_status, etc incorrectly. */ + if (exitcode == SIGUSR1) { + bysignal = SIGUSR1; + exitcode = 1; + } + if (pid == -1) { serverLog(LL_WARNING,"wait3() returned an error: %s. " "rdb_child_pid = %d, aof_child_pid = %d, module_child_pid = %d", @@ -4578,7 +4585,7 @@ static void sigKillChildHandler(int sig) { UNUSED(sig); /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(1); + exitFromChild(SIGUSR1); } void setupChildSignalHandlers(void) { diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 58a098edc..b364291ee 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -115,3 +115,17 @@ start_server_and_kill_it [list "dir" $server_path] { } } } + +start_server {} { + test {Test FLUSHALL aborts bgsave} { + r config set rdb-key-save-delay 1000 + r debug populate 1000 + r bgsave + assert_equal [s rdb_bgsave_in_progress] 1 + r flushall + after 200 + assert_equal [s rdb_bgsave_in_progress] 0 + # make sure the server is still writable + r set x xx + } +} \ No newline at end of file From 5d78cf769c005151abe15f77dbf8dd7a4393aa6b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Sep 2019 16:14:21 +0200 Subject: [PATCH 124/320] BGREWRITEAOF: improve the generic error message. --- src/aof.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 7237cdfbc..91d0d1a76 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1630,7 +1630,8 @@ void bgrewriteaofCommand(client *c) { } else if (rewriteAppendOnlyFileBackground() == C_OK) { addReplyStatus(c,"Background append only file rewriting started"); } else { - addReply(c,shared.err); + addReplyError(c,"Can't execute an AOF background rewriting. " + "Please check the server logs for more information."); } } From c9cf8a3c5f790d4297a209af4d6c1ea1b04b2139 Mon Sep 17 00:00:00 2001 From: nikhilajayk Date: Thu, 26 Sep 2019 21:51:49 +0530 Subject: [PATCH 125/320] Added cluster host and protected mode variables --- utils/create-cluster/README | 2 +- utils/create-cluster/create-cluster | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/create-cluster/README b/utils/create-cluster/README index e682f6dc9..37a3080db 100644 --- a/utils/create-cluster/README +++ b/utils/create-cluster/README @@ -16,7 +16,7 @@ To create a cluster, follow these steps: number of instances you want to create. 2. Use "./create-cluster start" in order to run the instances. 3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that -an actual Redis cluster will be created. +an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP) 4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory. In order to stop a cluster: diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 468f924a4..9ffd462ae 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -1,10 +1,12 @@ #!/bin/bash # Settings +CLUSTER_HOST=127.0.0.1 PORT=30000 TIMEOUT=2000 NODES=6 REPLICAS=1 +PROTECTED_MODE=yes # You may want to put the above config parameters into config.sh in order to # override the defaults without modifying this script. @@ -22,7 +24,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes + ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes done exit 0 fi @@ -32,7 +34,7 @@ then HOSTS="" while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) - HOSTS="$HOSTS 127.0.0.1:$PORT" + HOSTS="$HOSTS $CLUSTER_HOST:$PORT" done ../../src/redis-cli --cluster create $HOSTS --cluster-replicas $REPLICAS exit 0 From e885d7401882f4773fc3bad53eaba02e9a558dcd Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:39:40 +0200 Subject: [PATCH 126/320] Modules fork: improve SIGUSR1 handling, fix include. We can't expect SIGUSR1 to have any specific value range, so let's define an exit code that we can handle in a special way. This also fixes an #include that is not standard. --- src/module.c | 2 +- src/server.c | 14 ++++++++++---- src/server.h | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 854989e73..d3b37a3d8 100644 --- a/src/module.c +++ b/src/module.c @@ -31,7 +31,7 @@ #include "cluster.h" #include "rdb.h" #include -#include +#include #define REDISMODULE_CORE 1 #include "redismodule.h" diff --git a/src/server.c b/src/server.c index f38ed7897..046694e1f 100644 --- a/src/server.c +++ b/src/server.c @@ -1913,8 +1913,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); /* sigKillChildHandler catches the signal and calls exit(), but we - * must make sure not to flag lastbgsave_status, etc incorrectly. */ - if (exitcode == SIGUSR1) { + * must make sure not to flag lastbgsave_status, etc incorrectly. + * We could directly terminate the child process via SIGUSR1 + * without handling it, but in this case Valgrind will log an + * annoying error. */ + if (exitcode == SERVER_CHILD_NOERROR_RETVAL) { bysignal = SIGUSR1; exitcode = 1; } @@ -4618,11 +4621,14 @@ void setupSignalHandlers(void) { return; } +/* This is the signal handler for children process. It is currently useful + * in order to track the SIGUSR1, that we send to a child in order to terminate + * it in a clean way, without the parent detecting an error and stop + * accepting writes because of a write error condition. */ static void sigKillChildHandler(int sig) { UNUSED(sig); - /* this handler is needed to resolve a valgrind warning */ serverLogFromHandler(LL_WARNING, "Received SIGUSR1 in child, exiting now."); - exitFromChild(SIGUSR1); + exitFromChild(SERVER_CHILD_NOERROR_RETVAL); } void setupChildSignalHandlers(void) { diff --git a/src/server.h b/src/server.h index d132cf09c..c701d6d21 100644 --- a/src/server.h +++ b/src/server.h @@ -179,6 +179,14 @@ typedef long long mstime_t; /* millisecond time type. */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 +/* Children process will exit with this status code to signal that the + * process terminated without an error: this is useful in order to kill + * a saving child (RDB or AOF one), without triggering in the parent the + * write protection that is normally turned on on write errors. + * Usually children that are terminated with SIGUSR1 will exit with this + * special code. */ +#define SERVER_CHILD_NOERROR_RETVAL 255 + /* Instantaneous metrics tracking. */ #define STATS_METRIC_SAMPLES 16 /* Number of samples per metric. */ #define STATS_METRIC_COMMAND 0 /* Number of commands executed. */ From afc8866d661708364fa8f57dd65a2baa5e169a6b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:37 +0200 Subject: [PATCH 127/320] Improve error message in BGSAVE. --- src/rdb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index d9164b21c..e1cedf6f9 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2590,9 +2590,9 @@ void bgsaveCommand(client *c) { addReplyStatus(c,"Background saving scheduled"); } else { addReplyError(c, - "Another BG operation is in progress: can't BGSAVE right now. " - "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " - "possible."); + "Another child process is active (AOF?): can't BGSAVE right now. " + "Use BGSAVE SCHEDULE in order to schedule a BGSAVE whenever " + "possible."); } } else if (rdbSaveBackground(server.rdb_filename,rsiptr) == C_OK) { addReplyStatus(c,"Background saving started"); From eba38bc3a9317ada4dd7add999e60e8499dfff82 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 11:59:58 +0200 Subject: [PATCH 128/320] TerminateModuleForkChild(): use wait4 for safety. In theory currently there is only one active child, but the API may change or for bugs in the implementation we may have several (it was like that for years because of a bug). Better to wait for a specific pid and avoid consuing other pending children information. --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index d3b37a3d8..cee7a85a9 100644 --- a/src/module.c +++ b/src/module.c @@ -5193,7 +5193,8 @@ void TerminateModuleForkChild(int wait) { serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); if (kill(server.module_child_pid,SIGUSR1) != -1 && wait) { - while(wait3(&statloc,0,NULL) != server.module_child_pid); + while(wait4(server.module_child_pid,&statloc,0,NULL) != + server.module_child_pid); } /* Reset the buffer accumulating changes while the child saves. */ server.module_child_pid = -1; From 1ef0c2f6307297046b800b5f1c8a32b7efc5657a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:03:09 +0200 Subject: [PATCH 129/320] Function renamed hasForkChild() -> hasActiveChildProcess(). --- src/aof.c | 10 +++++----- src/db.c | 2 +- src/defrag.c | 2 +- src/module.c | 2 +- src/rdb.c | 6 +++--- src/replication.c | 4 ++-- src/server.c | 14 +++++++------- src/server.h | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/aof.c b/src/aof.c index 8bc6c543d..4e6af7c1c 100644 --- a/src/aof.c +++ b/src/aof.c @@ -264,7 +264,7 @@ int startAppendOnly(void) { strerror(errno)); return C_ERR; } - if (hasForkChild() && server.aof_child_pid == -1) { + if (hasActiveChildProcess() && server.aof_child_pid == -1) { server.aof_rewrite_scheduled = 1; serverLog(LL_WARNING,"AOF was enabled but there is already another background operation. An AOF background was scheduled to start when possible."); } else { @@ -395,7 +395,7 @@ void flushAppendOnlyFile(int force) { * useful for graphing / monitoring purposes. */ if (sync_in_progress) { latencyAddSampleIfNeeded("aof-write-pending-fsync",latency); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { latencyAddSampleIfNeeded("aof-write-active-child",latency); } else { latencyAddSampleIfNeeded("aof-write-alone",latency); @@ -491,7 +491,7 @@ void flushAppendOnlyFile(int force) { try_fsync: /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are * children doing I/O in the background. */ - if (server.aof_no_fsync_on_rewrite && hasForkChild()) + if (server.aof_no_fsync_on_rewrite && hasActiveChildProcess()) return; /* Perform the fsync if needed. */ @@ -1563,7 +1563,7 @@ void aofClosePipes(void) { int rewriteAppendOnlyFileBackground(void) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; if (aofCreatePipes() != C_OK) return C_ERR; openChildInfoPipe(); if ((childpid = redisFork()) == 0) { @@ -1607,7 +1607,7 @@ int rewriteAppendOnlyFileBackground(void) { void bgrewriteaofCommand(client *c) { if (server.aof_child_pid != -1) { addReplyError(c,"Background append only file rewriting already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { server.aof_rewrite_scheduled = 1; addReplyStatus(c,"Background append only file rewriting scheduled"); } else if (rewriteAppendOnlyFileBackground() == C_OK) { diff --git a/src/db.c b/src/db.c index a46e0251f..afedc6aec 100644 --- a/src/db.c +++ b/src/db.c @@ -60,7 +60,7 @@ robj *lookupKey(redisDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (!hasForkChild() && !(flags & LOOKUP_NOTOUCH)){ + if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)){ if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { updateLFU(val); } else { diff --git a/src/defrag.c b/src/defrag.c index 93c6a4619..50f6b41f2 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -1039,7 +1039,7 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; - if (hasForkChild()) + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ /* Once a second, check if we the fragmentation justfies starting a scan diff --git a/src/module.c b/src/module.c index cee7a85a9..4e6ad6cf9 100644 --- a/src/module.c +++ b/src/module.c @@ -5157,7 +5157,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; - if (hasForkChild()) { + if (hasActiveChildProcess()) { return -1; } diff --git a/src/rdb.c b/src/rdb.c index e1cedf6f9..e430bcd58 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1336,7 +1336,7 @@ werr: int rdbSaveBackground(char *filename, rdbSaveInfo *rsi) { pid_t childpid; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; server.dirty_before_bgsave = server.dirty; server.lastbgsave_try = time(NULL); @@ -2417,7 +2417,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { pid_t childpid; int pipefds[2]; - if (hasForkChild()) return C_ERR; + if (hasActiveChildProcess()) return C_ERR; /* Before to fork, create a pipe that will be used in order to * send back to the parent the IDs of the slaves that successfully @@ -2584,7 +2584,7 @@ void bgsaveCommand(client *c) { if (server.rdb_child_pid != -1) { addReplyError(c,"Background save already in progress"); - } else if (hasForkChild()) { + } else if (hasActiveChildProcess()) { if (schedule) { server.rdb_bgsave_scheduled = 1; addReplyStatus(c,"Background saving scheduled"); diff --git a/src/replication.c b/src/replication.c index 8039e06ae..76090a9e5 100644 --- a/src/replication.c +++ b/src/replication.c @@ -751,7 +751,7 @@ void syncCommand(client *c) { /* Target is disk (or the slave is not capable of supporting * diskless replication) and we don't have a BGSAVE in progress, * let's start one. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { startBgsaveForReplication(c->slave_capa); } else { serverLog(LL_NOTICE, @@ -2930,7 +2930,7 @@ void replicationCron(void) { * In case of diskless replication, we make sure to wait the specified * number of seconds (according to configuration) so that other slaves * have the time to arrive before we start streaming. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { time_t idle, max_idle = 0; int slaves_waiting = 0; int mincapa = -1; diff --git a/src/server.c b/src/server.c index 046694e1f..b2ab653a2 100644 --- a/src/server.c +++ b/src/server.c @@ -1449,13 +1449,13 @@ int incrementallyRehash(int dbid) { * for dict.c to resize the hash tables accordingly to the fact we have o not * running childs. */ void updateDictResizePolicy(void) { - if (!hasForkChild()) + if (!hasActiveChildProcess()) dictEnableResize(); else dictDisableResize(); } -int hasForkChild() { +int hasActiveChildProcess() { return server.rdb_child_pid != -1 || server.aof_child_pid != -1 || server.module_child_pid != -1; @@ -1697,7 +1697,7 @@ void databasesCron(void) { /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad * as will cause a lot of copy-on-write of memory pages. */ - if (!hasForkChild()) { + if (!hasActiveChildProcess()) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ @@ -1894,14 +1894,14 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.aof_rewrite_scheduled) { rewriteAppendOnlyFileBackground(); } /* Check if a background saving or AOF rewrite in progress terminated. */ - if (hasForkChild() || ldbPendingChildren()) + if (hasActiveChildProcess() || ldbPendingChildren()) { int statloc; pid_t pid; @@ -1975,7 +1975,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Trigger an AOF rewrite if needed. */ if (server.aof_state == AOF_ON && - !hasForkChild() && + !hasActiveChildProcess() && server.aof_rewrite_perc && server.aof_current_size > server.aof_rewrite_min_size) { @@ -2033,7 +2033,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * Note: this code must be after the replicationCron() call above so * make sure when refactoring this file to keep this order. This is useful * because we want to give priority to RDB savings for replication. */ - if (!hasForkChild() && + if (!hasActiveChildProcess() && server.rdb_bgsave_scheduled && (server.unixtime-server.lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY || server.lastbgsave_status == C_OK)) diff --git a/src/server.h b/src/server.h index c701d6d21..2ab3f2b45 100644 --- a/src/server.h +++ b/src/server.h @@ -1818,7 +1818,7 @@ void receiveChildInfo(void); /* Fork helpers */ int redisFork(); -int hasForkChild(); +int hasActiveChildProcess(); void sendChildCOWInfo(int ptype, char *pname); /* acl.c -- Authentication related prototypes. */ From 7aaedf9192c75ec1d502daa7277dae81dade6e97 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:17:47 +0200 Subject: [PATCH 130/320] TerminateModuleForkChild(): move safety checks there. We don't want that the API could be used directly in an unsafe way, without checking if there is an active child. Now the safety checks are moved directly in the function performing the operations. --- src/module.c | 35 ++++++++++++++++++----------------- src/server.c | 2 +- src/server.h | 2 +- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 4e6ad6cf9..27d5eb863 100644 --- a/src/module.c +++ b/src/module.c @@ -5031,7 +5031,7 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi ln = listSearchKey(moduleCommandFilters,filter); if (!ln) return REDISMODULE_ERR; listDelNode(moduleCommandFilters,ln); - + ln = listSearchKey(ctx->module->filters,filter); if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); @@ -5154,8 +5154,7 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) * Return: -1 on failure, on success the parent process will get a positive PID * of the child, and the child process will get 0. */ -int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) -{ +int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) { pid_t childpid; if (hasActiveChildProcess()) { return -1; @@ -5181,14 +5180,20 @@ int RM_Fork(RedisModuleForkDoneHandler cb, void *user_data) /* Call from the child process when you want to terminate it. * retcode will be provided to the done handler executed on the parent process. */ -int RM_ExitFromChild(int retcode) -{ +int RM_ExitFromChild(int retcode) { sendChildCOWInfo(CHILD_INFO_TYPE_MODULE, "Module fork"); exitFromChild(retcode); return REDISMODULE_OK; } -void TerminateModuleForkChild(int wait) { +/* Kill the active module forked child, if there is one active and the + * pid matches, and returns C_OK. Otherwise if there is no active module + * child or the pid does not match, return C_ERR without doing anything. */ +void TerminateModuleForkChild(int child_pid, int wait) { + /* Module child should be active and pid should match. */ + if (server.module_child_pid == -1 || + server.module_child_pid != child_pid) return C_ERR; + int statloc; serverLog(LL_NOTICE,"Killing running module fork child: %ld", (long) server.module_child_pid); @@ -5202,24 +5207,20 @@ void TerminateModuleForkChild(int wait) { moduleForkInfo.done_handler_user_data = NULL; closeChildInfoPipe(); updateDictResizePolicy(); + return C_OK; } /* Can be used to kill the forked child process from the parent process. * child_pid whould be the return value of RedisModule_Fork. */ -int RM_KillForkChild(int child_pid) -{ - /* No module child? return. */ - if (server.module_child_pid == -1) return REDISMODULE_ERR; - /* Make sure the module knows the pid it wants to kill (not trying to - * randomly kill other module's forks) */ - if (server.module_child_pid != child_pid) return REDISMODULE_ERR; +int RM_KillForkChild(int child_pid) { /* Kill module child, wait for child exit. */ - TerminateModuleForkChild(1); - return REDISMODULE_OK; + if (TerminateModuleForkChild(child_pid,1) == C_OK) + return REDISMODULE_OK; + else + return REDISMODULE_ERR; } -void ModuleForkDoneHandler(int exitcode, int bysignal) -{ +void ModuleForkDoneHandler(int exitcode, int bysignal) { serverLog(LL_NOTICE, "Module fork exited pid: %d, retcode: %d, bysignal: %d", server.module_child_pid, exitcode, bysignal); diff --git a/src/server.c b/src/server.c index b2ab653a2..a2af7824d 100644 --- a/src/server.c +++ b/src/server.c @@ -3589,7 +3589,7 @@ int prepareForShutdown(int flags) { /* Kill module child if there is one. */ if (server.module_child_pid != -1) { serverLog(LL_WARNING,"There is a module fork child. Killing it!"); - TerminateModuleForkChild(0); + TerminateModuleForkChild(server.module_child_pid,0); } if (server.aof_state != AOF_OFF) { diff --git a/src/server.h b/src/server.h index 2ab3f2b45..52abef4b8 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int wait); +void TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From 48aae0d0db1c029100d6c37c99f25bae911d93db Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 12:23:07 +0200 Subject: [PATCH 131/320] TerminateModuleForkChild(): fix function prototype. --- src/module.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 27d5eb863..217478705 100644 --- a/src/module.c +++ b/src/module.c @@ -5189,7 +5189,7 @@ int RM_ExitFromChild(int retcode) { /* Kill the active module forked child, if there is one active and the * pid matches, and returns C_OK. Otherwise if there is no active module * child or the pid does not match, return C_ERR without doing anything. */ -void TerminateModuleForkChild(int child_pid, int wait) { +int TerminateModuleForkChild(int child_pid, int wait) { /* Module child should be active and pid should match. */ if (server.module_child_pid == -1 || server.module_child_pid != child_pid) return C_ERR; diff --git a/src/server.h b/src/server.h index 52abef4b8..91c7219a0 100644 --- a/src/server.h +++ b/src/server.h @@ -1552,7 +1552,7 @@ void moduleReleaseGIL(void); void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid); void moduleCallCommandFilters(client *c); void ModuleForkDoneHandler(int exitcode, int bysignal); -void TerminateModuleForkChild(int child_pid, int wait); +int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); From 795d1029b4c54068ad32b82e53c03393b90672fe Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:32:52 +0200 Subject: [PATCH 132/320] Fix memory leak in RM_UnregisterCommandFilter(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 217478705..98d4db97f 100644 --- a/src/module.c +++ b/src/module.c @@ -5036,6 +5036,8 @@ int RM_UnregisterCommandFilter(RedisModuleCtx *ctx, RedisModuleCommandFilter *fi if (!ln) return REDISMODULE_ERR; /* Shouldn't happen */ listDelNode(ctx->module->filters,ln); + zfree(filter); + return REDISMODULE_OK; } From bd0ddcdc3bc652be02ca8f4f51a758acf3541d79 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:33:21 +0200 Subject: [PATCH 133/320] Fix memory leak in moduleLoadFromQueue(). --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index 98d4db97f..351ffcc45 100644 --- a/src/module.c +++ b/src/module.c @@ -5334,6 +5334,8 @@ void moduleLoadFromQueue(void) { void moduleFreeModuleStructure(struct RedisModule *module) { listRelease(module->types); listRelease(module->filters); + listRelease(module->usedby); + listRelease(module->using); sdsfree(module->name); zfree(module); } From 9ec77b6a7a9d9ec9283017a0ea354ad6a733faae Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Sep 2019 18:42:38 +0200 Subject: [PATCH 134/320] moduleRDBLoadError(): io->ctx may be NULL. The correct way to access the module about a given IO context is to deference io->type->module, since io->ctx is only populated if the user requests an explicit context from an IO object. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 351ffcc45..8e9ffbf87 100644 --- a/src/module.c +++ b/src/module.c @@ -3177,7 +3177,7 @@ void *RM_ModuleTypeGetValue(RedisModuleKey *key) { * modules this cannot be recovered, but if the module declared capability * to handle errors, we'll raise a flag rather than exiting. */ void moduleRDBLoadError(RedisModuleIO *io) { - if (io->ctx->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { + if (io->type->module->options & REDISMODULE_OPTIONS_HANDLE_IO_ERRORS) { io->error = 1; return; } From 0660ff3696b51e1b39fcf7416a57291748a6a5ef Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 10:58:15 +0200 Subject: [PATCH 135/320] Change a bit the style of #6385. --- src/module.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 9c2bed08f..13353df2c 100644 --- a/src/module.c +++ b/src/module.c @@ -1149,10 +1149,8 @@ int RM_ReplyWithLongLong(RedisModuleCtx *ctx, long long ll) { int replyWithStatus(RedisModuleCtx *ctx, const char *msg, char *prefix) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - const size_t msgLen = strlen(msg); - const size_t prefixLen = strlen(prefix); - addReplyProto(c,prefix,prefixLen); - addReplyProto(c,msg,msgLen); + addReplyProto(c,prefix,strlen(prefix)); + addReplyProto(c,msg,strlen(msg)); addReplyProto(c,"\r\n",2); return REDISMODULE_OK; } From 70dbce427369e8241850a62f63f356eecffdef38 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 17:17:36 +0200 Subject: [PATCH 136/320] Fix comments aesthetics. --- src/server.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index a2af7824d..79c458ee9 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,8 +4058,9 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* this is the total RSS overhead, including fragmentation, */ - mh->total_frag_bytes, /* named so for backwards compatibility */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation. */ + mh->total_frag_bytes, /* Named so for backwards compatibility. */ freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves, From 364c8601e39f3d4bc592e72b7c03a310e71ed050 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Tue, 17 Sep 2019 03:32:35 -0700 Subject: [PATCH 137/320] Allowed passing in of password hash and fixed config rewrite --- src/acl.c | 43 +++++++++++++++++++++++++++++++++++++----- tests/support/test.tcl | 6 ++++++ tests/unit/acl.tcl | 22 ++++++++++++++++++++- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/acl.c b/src/acl.c index 2cd729e77..9c5196f48 100644 --- a/src/acl.c +++ b/src/acl.c @@ -94,6 +94,9 @@ void ACLResetSubcommandsForCommand(user *u, unsigned long id); void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); +/* The length of the string representation of a hashed password. */ +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 + /* ============================================================================= * Helper functions for the rest of the ACL implementation * ==========================================================================*/ @@ -145,7 +148,7 @@ int time_independent_strcmp(char *a, char *b) { sds ACLHashPassword(unsigned char *cleartext, size_t len) { SHA256_CTX ctx; unsigned char hash[SHA256_BLOCK_SIZE]; - char hex[SHA256_BLOCK_SIZE*2]; + char hex[HASH_PASSWORD_LEN]; char *cset = "0123456789abcdef"; sha256_init(&ctx); @@ -156,7 +159,7 @@ sds ACLHashPassword(unsigned char *cleartext, size_t len) { hex[j*2] = cset[((hash[j]&0xF0)>>4)]; hex[j*2+1] = cset[(hash[j]&0xF)]; } - return sdsnewlen(hex,SHA256_BLOCK_SIZE*2); + return sdsnewlen(hex,HASH_PASSWORD_LEN); } /* ============================================================================= @@ -522,7 +525,7 @@ sds ACLDescribeUser(user *u) { listRewind(u->passwords,&li); while((ln = listNext(&li))) { sds thispass = listNodeValue(ln); - res = sdscatlen(res,">",1); + res = sdscatlen(res,"#",1); res = sdscatsds(res,thispass); res = sdscatlen(res," ",1); } @@ -649,6 +652,10 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). * < Remove this password from the list of valid passwords. * nopass All the set passwords of the user are removed, and the user * is flagged as requiring no password: it means that every @@ -685,6 +692,7 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * EEXIST: You are adding a key pattern after "*" was already added. This is * almost surely an error on the user side. * ENODEV: The password you are trying to remove from the user does not exist. + * EBADMSG: The hash you are trying to add is not a valid hash. */ int ACLSetUser(user *u, const char *op, ssize_t oplen) { if (oplen == -1) oplen = strlen(op); @@ -720,8 +728,30 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else if (!strcasecmp(op,"resetpass")) { u->flags &= ~USER_FLAG_NOPASS; listEmpty(u->passwords); - } else if (op[0] == '>') { - sds newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else if (op[0] == '>' || op[0] == '#') { + sds newpass; + if (op[0] == '>') { + newpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + + /* Password hashes can only be characters that represent + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. + */ + for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) { + char c = op[i]; + if ((c < 'a' || c > 'f') && (c < '0' || c > '9')) { + errno = EBADMSG; + return C_ERR; + } + } + newpass = sdsnewlen(op+1,oplen-1); + } + listNode *ln = listSearchKey(u->passwords,newpass); /* Avoid re-adding the same password multiple times. */ if (ln == NULL) @@ -848,6 +878,9 @@ char *ACLSetUserStringError(void) { else if (errno == ENODEV) errmsg = "The password you are trying to remove from the user does " "not exist"; + else if (errno == EBADMSG) + errmsg = "The password hash must be exactly 64 characters and contain " + "only lowercase hexadecimal characters"; return errmsg; } diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 6f02f2f12..2646acecd 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -15,6 +15,12 @@ proc assert {condition} { } } +proc assert_no_match {pattern value} { + if {[string match $pattern $value]} { + error "assertion:Expected '$value' to not match '$pattern'" + } +} + proc assert_match {pattern value} { if {![string match $pattern $value]} { error "assertion:Expected '$value' to match '$pattern'" diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index 058441433..d3b721205 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -35,6 +35,26 @@ start_server {tags {"acl"}} { set e } {*WRONGPASS*} + test {Test password hashes can be added} { + r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6 + catch {r AUTH newuser passwd4} e + assert {$e eq "OK"} + } + + test {Test password hashes validate input} { + # Validate Length + catch {r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e} e + # Validate character outside set + catch {r ACL setuser newuser #34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4eq} e + set e + } {*Error in ACL SETUSER modifier*} + + test {ACL GETUSER returns the password hash instead of the actual password} { + set passstr [dict get [r ACL getuser newuser] passwords] + assert_match {*34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6*} $passstr + assert_no_match {*passwd4*} $passstr + } + test {By default users are not able to access any command} { catch {r SET foo bar} e set e @@ -67,7 +87,7 @@ start_server {tags {"acl"}} { set e } {*NOPERM*} - test {ACLs can include or excluse whole classes of commands} { + test {ACLs can include or exclude whole classes of commands} { r ACL setuser newuser -@all +@set +acl r SADD myset a b c; # Should not raise an error r ACL setuser newuser +@all -@string From 611898979079153b192df1405e7dca9d8df712f6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 18:22:55 +0200 Subject: [PATCH 138/320] ACLs: change hashed passwords opcode to also remove them. Related to PR #6405 --- src/acl.c | 32 ++++++++++++++++++++++---------- tests/unit/acl.tcl | 6 ++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/acl.c b/src/acl.c index 9c5196f48..4c43add14 100644 --- a/src/acl.c +++ b/src/acl.c @@ -95,7 +95,7 @@ void ACLResetSubcommands(user *u); void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub); /* The length of the string representation of a hashed password. */ -#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 +#define HASH_PASSWORD_LEN SHA256_BLOCK_SIZE*2 /* ============================================================================= * Helper functions for the rest of the ACL implementation @@ -652,11 +652,14 @@ void ACLAddAllowedSubcommand(user *u, unsigned long id, const char *sub) { * > Add this password to the list of valid password for the user. * For example >mypass will add "mypass" to the list. * This directive clears the "nopass" flag (see later). - * # Add this password hash to the list of valid hashes for - * the user. This is useful if you have previously computed - * the hash, and don't want to store it in plaintext. - * This directive clears the "nopass" flag (see later). + * # Add this password hash to the list of valid hashes for + * the user. This is useful if you have previously computed + * the hash, and don't want to store it in plaintext. + * This directive clears the "nopass" flag (see later). * < Remove this password from the list of valid passwords. + * ! Remove this hashed password from the list of valid passwords. + * This is useful when you want to remove a password just by + * hash without knowing its plaintext version at all. * nopass All the set passwords of the user are removed, and the user * is flagged as requiring no password: it means that every * password will work against this user. If this directive is @@ -735,12 +738,12 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { } else { if (oplen != HASH_PASSWORD_LEN + 1) { errno = EBADMSG; - return C_ERR; + return C_ERR; } /* Password hashes can only be characters that represent - * hexadecimal values, which are numbers and lowercase - * characters 'a' through 'f'. + * hexadecimal values, which are numbers and lowercase + * characters 'a' through 'f'. */ for(int i = 1; i < HASH_PASSWORD_LEN + 1; i++) { char c = op[i]; @@ -759,8 +762,17 @@ int ACLSetUser(user *u, const char *op, ssize_t oplen) { else sdsfree(newpass); u->flags &= ~USER_FLAG_NOPASS; - } else if (op[0] == '<') { - sds delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else if (op[0] == '<' || op[0] == '!') { + sds delpass; + if (op[0] == '<') { + delpass = ACLHashPassword((unsigned char*)op+1,oplen-1); + } else { + if (oplen != HASH_PASSWORD_LEN + 1) { + errno = EBADMSG; + return C_ERR; + } + delpass = sdsnewlen(op+1,oplen-1); + } listNode *ln = listSearchKey(u->passwords,delpass); sdsfree(delpass); if (ln) { diff --git a/tests/unit/acl.tcl b/tests/unit/acl.tcl index d3b721205..2205d2d86 100644 --- a/tests/unit/acl.tcl +++ b/tests/unit/acl.tcl @@ -55,6 +55,12 @@ start_server {tags {"acl"}} { assert_no_match {*passwd4*} $passstr } + test {Test hashed passwords removal} { + r ACL setuser newuser !34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6 + set passstr [dict get [r ACL getuser newuser] passwords] + assert_no_match {*34344e4d60c2b6d639b7bd22e18f2b0b91bc34bf0ac5f9952744435093cfb4e6*} $passstr + } + test {By default users are not able to access any command} { catch {r SET foo bar} e set e From 700cc17a6a5f456932c7bab072319cd82bced2c6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2019 18:37:59 +0200 Subject: [PATCH 139/320] Clarify a comment about memory total_frag field. --- src/server.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 79c458ee9..cacce6dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -4058,9 +4058,11 @@ sds genRedisInfoString(char *section) { mh->allocator_rss_bytes, mh->rss_extra, mh->rss_extra_bytes, - mh->total_frag, /* This is the total RSS overhead, including - fragmentation. */ - mh->total_frag_bytes, /* Named so for backwards compatibility. */ + mh->total_frag, /* This is the total RSS overhead, including + fragmentation, but not just it. This field + (and the next one) is named like that just + for backward compatibility. */ + mh->total_frag_bytes, freeMemoryGetNotCountedMemory(), mh->repl_backlog, mh->clients_slaves, From 3556b6109cc44a805bf4608020eb9babc02eebf5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 30 Sep 2019 21:13:13 +0300 Subject: [PATCH 140/320] Use sdscatfmt instead of sdscatprintf in module info sdscatfmt is faster --- src/module.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/module.c b/src/module.c index efbf8eb83..18622bd6b 100644 --- a/src/module.c +++ b/src/module.c @@ -4824,7 +4824,7 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx); int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { sds full_name = sdsdup(ctx->module->name); if (name != NULL && strlen(name) > 0) - full_name = sdscatprintf(full_name, "_%s", name); + full_name = sdscatfmt(full_name, "_%s", name); /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) @@ -4843,7 +4843,7 @@ int RM_InfoAddSection(RedisModuleInfoCtx *ctx, char *name) { } } if (ctx->sections++) ctx->info = sdscat(ctx->info,"\r\n"); - ctx->info = sdscatprintf(ctx->info, "# %s\r\n", full_name); + ctx->info = sdscatfmt(ctx->info, "# %S\r\n", full_name); ctx->in_section = 1; sdsfree(full_name); return REDISMODULE_OK; @@ -4858,7 +4858,7 @@ int RM_InfoBeginDictField(RedisModuleInfoCtx *ctx, char *name) { /* Implicitly end dicts, instead of returning an error which is likely un checked. */ if (ctx->in_dict_field) RM_InfoEndDictField(ctx); - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s_%s:", ctx->module->name, name); @@ -4873,7 +4873,7 @@ int RM_InfoEndDictField(RedisModuleInfoCtx *ctx) { /* trim the last ',' if found. */ if (ctx->info[sdslen(ctx->info)-1]==',') sdsIncrLen(ctx->info, -1); - ctx->info = sdscatprintf(ctx->info, "\r\n"); + ctx->info = sdscat(ctx->info, "\r\n"); ctx->in_dict_field = 0; return REDISMODULE_OK; } @@ -4885,14 +4885,14 @@ int RM_InfoAddFieldString(RedisModuleInfoCtx *ctx, char *field, RedisModuleStrin if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%s,", + ctx->info = sdscatfmt(ctx->info, + "%s=%S,", field, (sds)value->ptr); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%s\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%S\r\n", ctx->module->name, field, (sds)value->ptr); @@ -4903,13 +4903,13 @@ int RM_InfoAddFieldCString(RedisModuleInfoCtx *ctx, char *field, char *value) { if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s=%s,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, + ctx->info = sdscatfmt(ctx->info, "%s_%s:%s\r\n", ctx->module->name, field, @@ -4939,14 +4939,14 @@ int RM_InfoAddFieldLongLong(RedisModuleInfoCtx *ctx, char *field, long long valu if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%lld,", + ctx->info = sdscatfmt(ctx->info, + "%s=%I,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%lld\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%I\r\n", ctx->module->name, field, value); @@ -4957,14 +4957,14 @@ int RM_InfoAddFieldULongLong(RedisModuleInfoCtx *ctx, char *field, unsigned long if (!ctx->in_section) return REDISMODULE_ERR; if (ctx->in_dict_field) { - ctx->info = sdscatprintf(ctx->info, - "%s=%llu,", + ctx->info = sdscatfmt(ctx->info, + "%s=%U,", field, value); return REDISMODULE_OK; } - ctx->info = sdscatprintf(ctx->info, - "%s_%s:%llu\r\n", + ctx->info = sdscatfmt(ctx->info, + "%s_%s:%U\r\n", ctx->module->name, field, value); @@ -5712,9 +5712,9 @@ sds genModulesInfoString(sds info) { sds usedby = genModulesInfoStringRenderModulesList(module->usedby); sds using = genModulesInfoStringRenderModulesList(module->using); sds options = genModulesInfoStringRenderModuleOptions(module); - info = sdscatprintf(info, - "module:name=%s,ver=%d,api=%d,filters=%d," - "usedby=%s,using=%s,options=%s\r\n", + info = sdscatfmt(info, + "module:name=%S,ver=%i,api=%i,filters=%i," + "usedby=%S,using=%S,options=%S\r\n", name, module->ver, module->apiver, (int)listLength(module->filters), usedby, using, options); sdsfree(usedby); From 0338f6b5c3da152b88af3cf78cba62e103f5386f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 1 Oct 2019 10:38:56 +0200 Subject: [PATCH 141/320] Fix GEORADIUS replies broken after RESP3 introduction. This commit fixes #6417. --- src/geo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geo.c b/src/geo.c index 826d11ff5..470a615f4 100644 --- a/src/geo.c +++ b/src/geo.c @@ -466,7 +466,7 @@ void georadiusGeneric(client *c, int flags) { /* Look up the requested zset */ robj *zobj = NULL; - if ((zobj = lookupKeyReadOrReply(c, key, shared.null[c->resp])) == NULL || + if ((zobj = lookupKeyReadOrReply(c, key, shared.emptyarray)) == NULL || checkType(c, zobj, OBJ_ZSET)) { return; } @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReplyNull(c); + addReplyNullArray(c); geoArrayFree(ga); return; } From c8b49a27aedbd4f367d67f3af52bb94108b5a16f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 1 Oct 2019 19:18:08 +0200 Subject: [PATCH 142/320] GEORADIUS reply: fix of the previous fix about #6417. --- src/geo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geo.c b/src/geo.c index 470a615f4..f47f4ee22 100644 --- a/src/geo.c +++ b/src/geo.c @@ -566,7 +566,7 @@ void georadiusGeneric(client *c, int flags) { /* If no matching results, the user gets an empty reply. */ if (ga->used == 0 && storekey == NULL) { - addReplyNullArray(c); + addReply(c,shared.emptyarray); geoArrayFree(ga); return; } From 497cfb61d65a63680771f3af636690b8843e6708 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 2 Oct 2019 08:40:35 +0300 Subject: [PATCH 143/320] On LUA script timeout, print the script SHA to the log since the slowlog and other means that can help you detect the bad script are only exposed after the script is done. it might be a good idea to at least print the script name (sha) to the log when it timeouts. --- src/scripting.c | 9 ++++++++- src/server.h | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 3129e4f47..8ce090e9c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1083,6 +1083,7 @@ void scriptingInit(int setup) { if (setup) { server.lua_client = NULL; server.lua_caller = NULL; + server.lua_cur_script = NULL; server.lua_timedout = 0; ldbInit(); } @@ -1407,7 +1408,11 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { /* Set the timeout condition if not already set and the maximum * execution time was reached. */ if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { - serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); + serverLog(LL_WARNING, + "Lua slow script detected: still in execution after %lld milliseconds. " + "You can try killing the script using the SCRIPT KILL command. " + "script SHA is: %s", + elapsed, server.lua_cur_script); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others * to call SCRIPT KILL or SHUTDOWN NOSAVE if needed. For this reason @@ -1524,6 +1529,7 @@ void evalGenericCommand(client *c, int evalsha) { * If we are debugging, we set instead a "line" hook so that the * debugger is call-back at every line executed by the script. */ server.lua_caller = c; + server.lua_cur_script = funcname + 2; server.lua_time_start = mstime(); server.lua_kill = 0; if (server.lua_time_limit > 0 && ldb.active == 0) { @@ -1550,6 +1556,7 @@ void evalGenericCommand(client *c, int evalsha) { queueClientForReprocessing(server.master); } server.lua_caller = NULL; + server.lua_cur_script = NULL; /* Call the Lua garbage collector from time to time to avoid a * full cycle performed by Lua, which adds too latency. diff --git a/src/server.h b/src/server.h index 6e011a2ca..6f3451eb7 100644 --- a/src/server.h +++ b/src/server.h @@ -1389,6 +1389,7 @@ struct redisServer { lua_State *lua; /* The Lua interpreter. We use just one for all clients */ client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ + char* lua_cur_script; /* The current script right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ From 22caeb868840cda3cb9d3133381ae5b7c3742e99 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 11:27:21 +0200 Subject: [PATCH 144/320] SDS: make sdscatfmt() faster by pre-allocating a bit. --- src/sds.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sds.c b/src/sds.c index cd60946bd..98bd2e77f 100644 --- a/src/sds.c +++ b/src/sds.c @@ -603,6 +603,10 @@ sds sdscatfmt(sds s, char const *fmt, ...) { long i; va_list ap; + /* To avoid continuous reallocations, let's start with a buffer that + * can hold at least two times the format string itself. It's not the + * best heuristic but seems to work in practice. */ + s = sdsMakeRoomFor(s, initlen + strlen(fmt)*2); va_start(ap,fmt); f = fmt; /* Next format specifier byte to process. */ i = initlen; /* Position of the next byte to write to dest str. */ From c1855c1053c690e68acbd00a3a318af54fbd90cc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 11:21:24 +0200 Subject: [PATCH 145/320] Speedup INFO server section. --- src/release.c | 14 ++++++++++++++ src/server.c | 34 +++++++++++++++++----------------- src/server.h | 1 + 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/src/release.c b/src/release.c index 4e59c7474..e0bd018fc 100644 --- a/src/release.c +++ b/src/release.c @@ -32,6 +32,7 @@ * files using this functions. */ #include +#include #include "release.h" #include "version.h" @@ -50,3 +51,16 @@ uint64_t redisBuildId(void) { return crc64(0,(unsigned char*)buildid,strlen(buildid)); } + +/* Return a cached value of the build string in order to avoid recomputing + * and converting it in hex every time: this string is shown in the INFO + * output that should be fast. */ +char *redisBuildIdString(void) { + static char buf[32]; + static int cached = 0; + if (!cached) { + snprintf(buf,sizeof(buf),"%llx",(unsigned long long) redisBuildId()); + cached = 1; + } + return buf; +} diff --git a/src/server.c b/src/server.c index 9380a840a..593f98f3f 100644 --- a/src/server.c +++ b/src/server.c @@ -3892,32 +3892,32 @@ sds genRedisInfoString(char *section) { call_uname = 0; } - info = sdscatprintf(info, + info = sdscatfmt(info, "# Server\r\n" "redis_version:%s\r\n" "redis_git_sha1:%s\r\n" - "redis_git_dirty:%d\r\n" - "redis_build_id:%llx\r\n" + "redis_git_dirty:%i\r\n" + "redis_build_id:%s\r\n" "redis_mode:%s\r\n" "os:%s %s %s\r\n" - "arch_bits:%d\r\n" + "arch_bits:%i\r\n" "multiplexing_api:%s\r\n" "atomicvar_api:%s\r\n" - "gcc_version:%d.%d.%d\r\n" - "process_id:%ld\r\n" + "gcc_version:%i.%i.%i\r\n" + "process_id:%I\r\n" "run_id:%s\r\n" - "tcp_port:%d\r\n" - "uptime_in_seconds:%jd\r\n" - "uptime_in_days:%jd\r\n" - "hz:%d\r\n" - "configured_hz:%d\r\n" - "lru_clock:%ld\r\n" + "tcp_port:%i\r\n" + "uptime_in_seconds:%I\r\n" + "uptime_in_days:%I\r\n" + "hz:%i\r\n" + "configured_hz:%i\r\n" + "lru_clock:%u\r\n" "executable:%s\r\n" "config_file:%s\r\n", REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, - (unsigned long long) redisBuildId(), + redisBuildIdString(), mode, name.sysname, name.release, name.machine, server.arch_bits, @@ -3928,14 +3928,14 @@ sds genRedisInfoString(char *section) { #else 0,0,0, #endif - (long) getpid(), + (int64_t) getpid(), server.runid, server.port, - (intmax_t)uptime, - (intmax_t)(uptime/(3600*24)), + (int64_t)uptime, + (int64_t)(uptime/(3600*24)), server.hz, server.config_hz, - (unsigned long) server.lruclock, + server.lruclock, server.executable ? server.executable : "", server.configfile ? server.configfile : ""); } diff --git a/src/server.h b/src/server.h index 6e011a2ca..4d48fcd1a 100644 --- a/src/server.h +++ b/src/server.h @@ -2140,6 +2140,7 @@ void dictSdsDestructor(void *privdata, void *val); char *redisGitSHA1(void); char *redisGitDirty(void); uint64_t redisBuildId(void); +char *redisBuildIdString(void); /* Commands prototypes */ void authCommand(client *c); From 086b3911073c9cc41bd0f998764fa592384c9a7e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 9 Sep 2019 14:35:06 +0300 Subject: [PATCH 146/320] RED-31295 - redis: avoid race between dlopen and thread creation It seeems that since I added the creation of the jemalloc thread redis sometimes fails to start with the following error: Inconsistency detected by ld.so: dl-tls.c: 493: _dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= GL(dl_tls_generation)' failed! This seems to be due to a race bug in ld.so, in which TLS creation on the thread, collide with dlopen. Move the creation of BIO and jemalloc threads to after modules are loaded. plus small bugfix when trying to disable the jemalloc thread at runtime --- src/server.c | 10 ++++++++++ src/zmalloc.c | 6 ++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index fa2c7b1ee..8e99431f9 100644 --- a/src/server.c +++ b/src/server.c @@ -2865,6 +2865,14 @@ void initServer(void) { scriptingInit(1); slowlogInit(); latencyMonitorInit(); +} + +/* Some steps in server initialization need to be done last (after modules + * are loaded). + * Specifically, creation of threads due to a race bug in ld.so, in which + * Thread Local Storage initialization collides with dlopen call. + * see: https://sourceware.org/bugzilla/show_bug.cgi?id=19329 */ +void InitServerLast() { bioInit(); initThreadedIO(); set_jemalloc_bg_thread(server.jemalloc_bg_thread); @@ -4876,6 +4884,7 @@ int main(int argc, char **argv) { #endif moduleLoadFromQueue(); ACLLoadUsersAtStartup(); + InitServerLast(); loadDataFromDisk(); if (server.cluster_enabled) { if (verifyClusterConfigWithData() == C_ERR) { @@ -4890,6 +4899,7 @@ int main(int argc, char **argv) { if (server.sofd > 0) serverLog(LL_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); } else { + InitServerLast(); sentinelIsRunning(); } diff --git a/src/zmalloc.c b/src/zmalloc.c index 58896a727..437f43b64 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -332,10 +332,8 @@ int zmalloc_get_allocator_info(size_t *allocated, void set_jemalloc_bg_thread(int enable) { /* let jemalloc do purging asynchronously, required when there's no traffic * after flushdb */ - if (enable) { - char val = 1; - je_mallctl("background_thread", NULL, 0, &val, 1); - } + char val = !!enable; + je_mallctl("background_thread", NULL, 0, &val, 1); } int jemalloc_purge() { From 5526c0b9cb58e0e7934ab716268d28929b7e6a25 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 Oct 2019 18:33:40 +0200 Subject: [PATCH 147/320] Modules: handle propagation when ctx is freed. Flag modules commands ctx. --- src/module.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 18622bd6b..9876a966d 100644 --- a/src/module.c +++ b/src/module.c @@ -158,6 +158,7 @@ typedef struct RedisModuleCtx RedisModuleCtx; #define REDISMODULE_CTX_BLOCKED_TIMEOUT (1<<4) #define REDISMODULE_CTX_THREAD_SAFE (1<<5) #define REDISMODULE_CTX_BLOCKED_DISCONNECTED (1<<6) +#define REDISMODULE_CTX_MODULE_COMMAND_CALL (1<<7) /* This represents a Redis key opened with RM_OpenKey(). */ struct RedisModuleKey { @@ -519,8 +520,29 @@ int RM_GetApi(const char *funcname, void **targetPtrPtr) { return REDISMODULE_OK; } +/* Helper function for when a command callback is called, in order to handle + * details needed to correctly replicate commands. */ +void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { + client *c = ctx->client; + + /* We don't need to do anything here if the context was never used + * in order to propagate commands. */ + if (!(ctx->flags & REDISMODULE_CTX_MULTI_EMITTED)) return; + + if (c->flags & CLIENT_LUA) return; + + /* Handle the replication of the final EXEC, since whatever a command + * emits is always wrapped around MULTI/EXEC. */ + robj *propargv[1]; + propargv[0] = createStringObject("EXEC",4); + alsoPropagate(server.execCommand,c->db->id,propargv,1, + PROPAGATE_AOF|PROPAGATE_REPL); + decrRefCount(propargv[0]); +} + /* Free the context after the user function was called. */ void moduleFreeContext(RedisModuleCtx *ctx) { + moduleHandlePropagationAfterCommandCallback(ctx); autoMemoryCollect(ctx); poolAllocRelease(ctx); if (ctx->postponed_arrays) { @@ -536,34 +558,16 @@ void moduleFreeContext(RedisModuleCtx *ctx) { if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) freeClient(ctx->client); } -/* Helper function for when a command callback is called, in order to handle - * details needed to correctly replicate commands. */ -void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { - client *c = ctx->client; - - if (c->flags & CLIENT_LUA) return; - - /* Handle the replication of the final EXEC, since whatever a command - * emits is always wrapped around MULTI/EXEC. */ - if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) { - robj *propargv[1]; - propargv[0] = createStringObject("EXEC",4); - alsoPropagate(server.execCommand,c->db->id,propargv,1, - PROPAGATE_AOF|PROPAGATE_REPL); - decrRefCount(propargv[0]); - } -} - /* This Redis command binds the normal Redis command invocation with commands * exported by modules. */ void RedisModuleCommandDispatcher(client *c) { RedisModuleCommandProxy *cp = (void*)(unsigned long)c->cmd->getkeys_proc; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_MODULE_COMMAND_CALL; ctx.module = cp->module; ctx.client = c; cp->func(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); /* In some cases processMultibulkBuffer uses sdsMakeRoomFor to From 67cdd03efa0c35d99047d6f4231b172432c04985 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 10:56:37 +0200 Subject: [PATCH 148/320] Modules: implement RM_Replicate() from async callbacks. --- src/module.c | 33 +++++++++++++++++++++++++++++++-- src/server.h | 2 ++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 9876a966d..e8fb5adb3 100644 --- a/src/module.c +++ b/src/module.c @@ -147,10 +147,14 @@ struct RedisModuleCtx { int keys_count; struct RedisModulePoolAllocBlock *pa_head; + redisOpArray saved_oparray; /* When propagating commands in a callback + we reallocate the "also propagate" op + array. Here we save the old one to + restore it later. */ }; typedef struct RedisModuleCtx RedisModuleCtx; -#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL} +#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL, {0}} #define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) @@ -538,6 +542,24 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { alsoPropagate(server.execCommand,c->db->id,propargv,1, PROPAGATE_AOF|PROPAGATE_REPL); decrRefCount(propargv[0]); + + /* If this is not a module command context (but is instead a simple + * callback context), we have to handle directly the "also propagate" + * array and emit it. In a module command call this will be handled + * directly by call(). */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL) && + server.also_propagate.numops) + { + for (int j = 0; j < server.also_propagate.numops; j++) { + redisOp *rop = &server.also_propagate.ops[j]; + int target = rop->target; + if (target) + propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target); + } + redisOpArrayFree(&server.also_propagate); + } + /* Restore the previous oparray in case of nexted use of the API. */ + server.also_propagate = ctx->saved_oparray; } /* Free the context after the user function was called. */ @@ -1354,9 +1376,16 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { /* If we already emitted MULTI return ASAP. */ if (ctx->flags & REDISMODULE_CTX_MULTI_EMITTED) return; /* If this is a thread safe context, we do not want to wrap commands - * executed into MUTLI/EXEC, they are executed as single commands + * executed into MULTI/EXEC, they are executed as single commands * from an external client in essence. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) return; + /* If this is a callback context, and not a module command execution + * context, we have to setup the op array for the "also propagate" API + * so that RM_Replicate() will work. */ + if (!(ctx->flags & REDISMODULE_CTX_MODULE_COMMAND_CALL)) { + ctx->saved_oparray = server.also_propagate; + redisOpArrayInit(&server.also_propagate); + } execCommandPropagateMulti(ctx->client); ctx->flags |= REDISMODULE_CTX_MULTI_EMITTED; } diff --git a/src/server.h b/src/server.h index 4d48fcd1a..c78010ecc 100644 --- a/src/server.h +++ b/src/server.h @@ -1922,6 +1922,8 @@ struct redisCommand *lookupCommandOrOriginal(sds name); void call(client *c, int flags); void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags); void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int target); +void redisOpArrayInit(redisOpArray *oa); +void redisOpArrayFree(redisOpArray *oa); void forceCommandPropagation(client *c, int flags); void preventCommandPropagation(client *c); void preventCommandAOF(client *c); From 414f633962f11400089d4965b3f82a2789da06ca Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 11:03:46 +0200 Subject: [PATCH 149/320] Modules: RM_Replicate() in thread safe contexts. --- src/module.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index e8fb5adb3..c87f81c5d 100644 --- a/src/module.c +++ b/src/module.c @@ -1407,6 +1407,20 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * ## Note about calling this function from a thread safe context: + * + * Normally when you call this function from the callback implementing a + * module command, or any other callback provided by the Redis Module API, + * Redis will accumulate all the calls to this function in the context of + * the callback, and will propagate all the commands wrapped in a MULTI/EXEC + * transaction. However when calling this function from a threaded safe context + * that can live an undefined amount of time, and can be locked/unlocked in + * at will, the behavior is different: MULTI/EXEC wrapper is not emitted + * and the command specified is inserted in the AOF and replication stream + * immediately. + * + * ## Return value + * * The command returns REDISMODULE_ERR if the format specifiers are invalid * or the command name does not belong to a known command. */ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { @@ -1424,10 +1438,18 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; - /* Replicate! */ - moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + /* Replicate! When we are in a threaded context, we want to just insert + * the replicated command ASAP, since it is not clear when the context + * will stop being used, so accumulating stuff does not make much sense, + * nor we could easily use the alsoPropagate() API from threads. */ + if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { + propagate(cmd,ctx->client->db->id,argv,argc, + PROPAGATE_AOF|PROPAGATE_REPL); + } else { + moduleReplicateMultiIfNeeded(ctx); + alsoPropagate(cmd,ctx->client->db->id,argv,argc, + PROPAGATE_AOF|PROPAGATE_REPL); + } /* Release the argv. */ for (j = 0; j < argc; j++) decrRefCount(argv[j]); From d859b4289958959c94ab6cda11fd64fe6a75e37d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 13:06:09 +0200 Subject: [PATCH 150/320] Modules: RM_Replicate() test module: initial implementation. --- tests/modules/Makefile | 5 ++- tests/modules/propagate.c | 82 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 tests/modules/propagate.c diff --git a/tests/modules/Makefile b/tests/modules/Makefile index e669d8e5e..c1c8ffa2e 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so testrdb.so fork.so infotest.so +all: commandfilter.so testrdb.so fork.so infotest.so propagate.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -22,6 +22,7 @@ commandfilter.xo: ../../src/redismodule.h fork.xo: ../../src/redismodule.h testrdb.xo: ../../src/redismodule.h infotest.xo: ../../src/redismodule.h +propagate.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc @@ -35,3 +36,5 @@ testrdb.so: testrdb.xo infotest.so: infotest.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc +propagate.so: propagate.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c new file mode 100644 index 000000000..e08372dcf --- /dev/null +++ b/tests/modules/propagate.c @@ -0,0 +1,82 @@ +/* This module is used to test the propagation (replication + AOF) of + * commands, via the RedisModule_Replicate() interface, in asynchronous + * contexts, such as callbacks not implementing commands, and thread safe + * contexts. + * + * We create a timer callback and a threads using a thread safe context. + * Using both we try to propagate counters increments, and later we check + * if the replica contains the changes as expected. + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +/* Timer callback. */ +void timerHandler(RedisModuleCtx *ctx, void *data) { + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(data); + + static int times = 0; + + printf("Fired!\n"); + RedisModule_Replicate(ctx,"INCR","c","timer"); + times++; + + if (times < 10) + RedisModule_CreateTimer(ctx,100,timerHandler,NULL); +} + +int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModuleTimerID tid = RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + REDISMODULE_NOT_USED(tid); + + RedisModule_ReplyWithSimpleString(ctx,"OK"); + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"propagate-test",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"propagate-test", + propagateTestCommand, + "",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; +} From d6d4fea080635a6b01856b2af8eecde2fab5bd74 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 13:23:48 +0200 Subject: [PATCH 151/320] Modules: RM_Replicate() test with threads. --- tests/modules/propagate.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index e08372dcf..fbd3ec359 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -39,6 +39,7 @@ #define REDISMODULE_EXPERIMENTAL_API #include "redismodule.h" +#include /* Timer callback. */ void timerHandler(RedisModuleCtx *ctx, void *data) { @@ -47,12 +48,26 @@ void timerHandler(RedisModuleCtx *ctx, void *data) { static int times = 0; - printf("Fired!\n"); RedisModule_Replicate(ctx,"INCR","c","timer"); times++; if (times < 10) RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + else + times = 0; +} + +/* The thread entry point. */ +void *threadMain(void *arg) { + REDISMODULE_NOT_USED(arg); + RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL); + for (int i = 0; i < 10; i++) { + RedisModule_ThreadSafeContextLock(ctx); + RedisModule_Replicate(ctx,"INCR","c","thread"); + RedisModule_ThreadSafeContextUnlock(ctx); + } + RedisModule_FreeThreadSafeContext(ctx); + return NULL; } int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) @@ -60,7 +75,13 @@ int propagateTestCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); - RedisModuleTimerID tid = RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + RedisModuleTimerID timer_id = + RedisModule_CreateTimer(ctx,100,timerHandler,NULL); + REDISMODULE_NOT_USED(timer_id); + + pthread_t tid; + if (pthread_create(&tid,NULL,threadMain,NULL) != 0) + return RedisModule_ReplyWithError(ctx,"-ERR Can't start thread"); REDISMODULE_NOT_USED(tid); RedisModule_ReplyWithSimpleString(ctx,"OK"); From 59982e85eb932d6c5623914691cb856ab842b1b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Oct 2019 18:44:47 +0200 Subject: [PATCH 152/320] Modules: add RM_Replicate() Tcl test file & fix the module. --- runtest-moduleapi | 2 +- tests/modules/propagate.c | 1 + tests/unit/moduleapi/propagate.tcl | 30 ++++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/unit/moduleapi/propagate.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 1f090ff65..a16cca686 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate "${@}" diff --git a/tests/modules/propagate.c b/tests/modules/propagate.c index fbd3ec359..f83af1799 100644 --- a/tests/modules/propagate.c +++ b/tests/modules/propagate.c @@ -61,6 +61,7 @@ void timerHandler(RedisModuleCtx *ctx, void *data) { void *threadMain(void *arg) { REDISMODULE_NOT_USED(arg); RedisModuleCtx *ctx = RedisModule_GetThreadSafeContext(NULL); + RedisModule_SelectDb(ctx,9); /* Tests ran in database number 9. */ for (int i = 0; i < 10; i++) { RedisModule_ThreadSafeContextLock(ctx); RedisModule_Replicate(ctx,"INCR","c","thread"); diff --git a/tests/unit/moduleapi/propagate.tcl b/tests/unit/moduleapi/propagate.tcl new file mode 100644 index 000000000..71307ce33 --- /dev/null +++ b/tests/unit/moduleapi/propagate.tcl @@ -0,0 +1,30 @@ +set testmodule [file normalize tests/modules/propagate.so] + +tags "modules" { + test {Modules can propagate in async and threaded contexts} { + start_server {} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server [list overrides [list loadmodule "$testmodule"]] { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + + # Start the replication process... + $replica replicaof $master_host $master_port + wait_for_sync $replica + + after 1000 + $master propagate-test + + wait_for_condition 5000 10 { + ([$replica get timer] eq "10") && \ + ([$replica get thread] eq "10") + } else { + fail "The two counters don't match the expected value." + } + } + } + } +} From 9f2a31368cd0164bb26a6ca4e5b87ecbd2a76a05 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 11:44:39 +0200 Subject: [PATCH 153/320] Modules: RM_Call/Replicate() ability to exclude AOF/replicas. --- src/module.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/module.c b/src/module.c index c87f81c5d..0e9c22e54 100644 --- a/src/module.c +++ b/src/module.c @@ -318,6 +318,10 @@ static struct RedisModuleForkInfo { void* done_handler_user_data; } moduleForkInfo = {0}; +/* Flags for moduleCreateArgvFromUserFormat(). */ +#define REDISMODULE_ARGV_REPLICATE (1<<0) +#define REDISMODULE_ARGV_NO_AOF (1<<1) +#define REDISMODULE_ARGV_NO_REPLICAS (1<<2) /* -------------------------------------------------------------------------- * Prototypes @@ -1407,6 +1411,10 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx) { * * Please refer to RedisModule_Call() for more information. * + * Using the special "A" and "R" modifiers, the caller can exclude either + * the AOF or the replicas from the propagation of the specified command. + * Otherwise, by default, the command will be propagated in both channels. + * * ## Note about calling this function from a thread safe context: * * Normally when you call this function from the callback implementing a @@ -1438,17 +1446,22 @@ int RM_Replicate(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) va_end(ap); if (argv == NULL) return REDISMODULE_ERR; + /* Select the propagation target. Usually is AOF + replicas, however + * the caller can exclude one or the other using the "A" or "R" + * modifiers. */ + int target = 0; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) target |= PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) target |= PROPAGATE_REPL; + /* Replicate! When we are in a threaded context, we want to just insert * the replicated command ASAP, since it is not clear when the context * will stop being used, so accumulating stuff does not make much sense, * nor we could easily use the alsoPropagate() API from threads. */ if (ctx->flags & REDISMODULE_CTX_THREAD_SAFE) { - propagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + propagate(cmd,ctx->client->db->id,argv,argc,target); } else { moduleReplicateMultiIfNeeded(ctx); - alsoPropagate(cmd,ctx->client->db->id,argv,argc, - PROPAGATE_AOF|PROPAGATE_REPL); + alsoPropagate(cmd,ctx->client->db->id,argv,argc,target); } /* Release the argv. */ @@ -2767,12 +2780,11 @@ RedisModuleString *RM_CreateStringFromCallReply(RedisModuleCallReply *reply) { * to special modifiers in "fmt". For now only one exists: * * "!" -> REDISMODULE_ARGV_REPLICATE + * "A" -> REDISMODULE_ARGV_NO_AOF + * "R" -> REDISMODULE_ARGV_NO_REPLICAS * * On error (format specifier error) NULL is returned and nothing is * allocated. On success the argument vector is returned. */ - -#define REDISMODULE_ARGV_REPLICATE (1<<0) - robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int *argcp, int *flags, va_list ap) { int argc = 0, argv_size, j; robj **argv = NULL; @@ -2821,6 +2833,10 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int } } else if (*p == '!') { if (flags) (*flags) |= REDISMODULE_ARGV_REPLICATE; + } else if (*p == 'A') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_AOF; + } else if (*p == 'R') { + if (flags) (*flags) |= REDISMODULE_ARGV_NO_REPLICAS; } else { goto fmterr; } @@ -2912,8 +2928,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* Run the command */ int call_flags = CMD_CALL_SLOWLOG | CMD_CALL_STATS; if (replicate) { - call_flags |= CMD_CALL_PROPAGATE_AOF; - call_flags |= CMD_CALL_PROPAGATE_REPL; + if (!(flags & REDISMODULE_ARGV_NO_AOF)) + call_flags |= CMD_CALL_PROPAGATE_AOF; + if (!(flags & REDISMODULE_ARGV_NO_REPLICAS)) + call_flags |= CMD_CALL_PROPAGATE_REPL; } call(c,call_flags); From 124493582b306b9fb0eb2067d319f46b8ad36215 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 11:46:53 +0200 Subject: [PATCH 154/320] Modules: RM_Call(): give pointer to documentation. --- src/module.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 0e9c22e54..82ba9f86c 100644 --- a/src/module.c +++ b/src/module.c @@ -2857,7 +2857,10 @@ fmterr: * NULL is returned and errno is set to the following values: * * EINVAL: command non existing, wrong arity, wrong format specifier. - * EPERM: operation in Cluster instance with key in non local slot. */ + * EPERM: operation in Cluster instance with key in non local slot. + * + * This API is documented here: https://redis.io/topics/modules-intro + */ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...) { struct redisCommand *cmd; client *c = NULL; From 46da87cdbbff1e5e5fbd9974b927de0afef69d2b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 12:00:41 +0200 Subject: [PATCH 155/320] Minor aesthetic changes to #6419. --- src/scripting.c | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 8ce090e9c..ec95eb256 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1411,7 +1411,7 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { serverLog(LL_WARNING, "Lua slow script detected: still in execution after %lld milliseconds. " "You can try killing the script using the SCRIPT KILL command. " - "script SHA is: %s", + "Script SHA1 is: %s", elapsed, server.lua_cur_script); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others diff --git a/src/server.h b/src/server.h index f24c428ca..2e5749907 100644 --- a/src/server.h +++ b/src/server.h @@ -1389,7 +1389,7 @@ struct redisServer { lua_State *lua; /* The Lua interpreter. We use just one for all clients */ client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ - char* lua_cur_script; /* The current script right now, or NULL */ + char* lua_cur_script; /* SHA1 of the script currently running, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ From f484baa3ea8254ba491422f8f358d36b9cfac236 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Fri, 4 Oct 2019 14:22:13 +0300 Subject: [PATCH 156/320] trim the double implementation of jemalloc purge --- src/object.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/src/object.c b/src/object.c index 697429b84..70022f897 100644 --- a/src/object.c +++ b/src/object.c @@ -1450,22 +1450,10 @@ NULL addReplyVerbatim(c,report,sdslen(report),"txt"); sdsfree(report); } else if (!strcasecmp(c->argv[1]->ptr,"purge") && c->argc == 2) { -#if defined(USE_JEMALLOC) - char tmp[32]; - unsigned narenas = 0; - size_t sz = sizeof(unsigned); - if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) { - sprintf(tmp, "arena.%d.purge", narenas); - if (!je_mallctl(tmp, NULL, 0, NULL, 0)) { - addReply(c, shared.ok); - return; - } - } - addReplyError(c, "Error purging dirty pages"); -#else - addReply(c, shared.ok); - /* Nothing to do for other allocators. */ -#endif + if (jemalloc_purge() == 0) + addReply(c, shared.ok); + else + addReplyError(c, "Error purging dirty pages"); } else { addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr); } From f30328c793950bb267a50d596d62a7433369a6d9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 18:52:07 +0200 Subject: [PATCH 157/320] LOLWUT: refactoring + skeleton of LOLWUT 6. --- src/Makefile | 2 +- src/lolwut.c | 121 +++++++++++++++++++++++++++++++++++++++++++++++++- src/lolwut.h | 49 ++++++++++++++++++++ src/lolwut5.c | 110 +-------------------------------------------- src/lolwut6.c | 75 +++++++++++++++++++++++++++++++ 5 files changed, 247 insertions(+), 110 deletions(-) create mode 100644 src/lolwut.h create mode 100644 src/lolwut6.c diff --git a/src/Makefile b/src/Makefile index 198d85cd5..a76adbf4f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -164,7 +164,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o sha256.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/lolwut.c b/src/lolwut.c index ba7e1069e..d3e8fbad5 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -34,8 +34,11 @@ */ #include "server.h" +#include "lolwut.h" +#include void lolwut5Command(client *c); +void lolwut6Command(client *c); /* The default target for LOLWUT if no matching version was found. * This is what unstable versions of Redis will display. */ @@ -47,11 +50,127 @@ void lolwutUnstableCommand(client *c) { sdsfree(rendered); } +/* LOLWUT [] */ void lolwutCommand(client *c) { char *v = REDIS_VERSION; - if ((v[0] == '5' && v[1] == '.') || + char verstr[64]; + + if (c->argc == 2) { + long ver; + if (getLongFromObjectOrReply(c,c->argv[1],&ver,NULL) != C_OK) return; + snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); + v = verstr; + } + + if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || (v[0] == '4' && v[1] == '.' && v[2] == '9')) lolwut5Command(c); + else if ((v[0] == '6' && v[1] == '.' && v[2] != '9') || + (v[0] == '5' && v[1] == '.' && v[2] == '9')) + lolwut6Command(c); else lolwutUnstableCommand(c); } + +/* ========================== LOLWUT Canvase =============================== + * Many LOWUT versions will likely print some computer art to the screen. + * This is the case with LOLWUT 5 and LOLWUT 6, so here there is a generic + * canvas implementation that can be reused. */ + +/* Allocate and return a new canvas of the specified size. */ +lwCanvas *lwCreateCanvas(int width, int height) { + lwCanvas *canvas = zmalloc(sizeof(*canvas)); + canvas->width = width; + canvas->height = height; + canvas->pixels = zmalloc(width*height); + memset(canvas->pixels,0,width*height); + return canvas; +} + +/* Free the canvas created by lwCreateCanvas(). */ +void lwFreeCanvas(lwCanvas *canvas) { + zfree(canvas->pixels); + zfree(canvas); +} + +/* Set a pixel to the specified color. Color is 0 or 1, where zero means no + * dot will be displyed, and 1 means dot will be displayed. + * Coordinates are arranged so that left-top corner is 0,0. You can write + * out of the size of the canvas without issues. */ +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return; + canvas->pixels[x+y*canvas->width] = color; +} + +/* Return the value of the specified pixel on the canvas. */ +int lwGetPixel(lwCanvas *canvas, int x, int y) { + if (x < 0 || x >= canvas->width || + y < 0 || y >= canvas->height) return 0; + return canvas->pixels[x+y*canvas->width]; +} + +/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { + int dx = abs(x2-x1); + int dy = abs(y2-y1); + int sx = (x1 < x2) ? 1 : -1; + int sy = (y1 < y2) ? 1 : -1; + int err = dx-dy, e2; + + while(1) { + lwDrawPixel(canvas,x1,y1,color); + if (x1 == x2 && y1 == y2) break; + e2 = err*2; + if (e2 > -dy) { + err -= dy; + x1 += sx; + } + if (e2 < dx) { + err += dx; + y1 += sy; + } + } +} + +/* Draw a square centered at the specified x,y coordinates, with the specified + * rotation angle and size. In order to write a rotated square, we use the + * trivial fact that the parametric equation: + * + * x = sin(k) + * y = cos(k) + * + * Describes a circle for values going from 0 to 2*PI. So basically if we start + * at 45 degrees, that is k = PI/4, with the first point, and then we find + * the other three points incrementing K by PI/2 (90 degrees), we'll have the + * points of the square. In order to rotate the square, we just start with + * k = PI/4 + rotation_angle, and we are done. + * + * Of course the vanilla equations above will describe the square inside a + * circle of radius 1, so in order to draw larger squares we'll have to + * multiply the obtained coordinates, and then translate them. However this + * is much simpler than implementing the abstract concept of 2D shape and then + * performing the rotation/translation transformation, so for LOLWUT it's + * a good approach. */ +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color) { + int px[4], py[4]; + + /* Adjust the desired size according to the fact that the square inscribed + * into a circle of radius 1 has the side of length SQRT(2). This way + * size becomes a simple multiplication factor we can use with our + * coordinates to magnify them. */ + size /= 1.4142135623; + size = round(size); + + /* Compute the four points. */ + float k = M_PI/4 + angle; + for (int j = 0; j < 4; j++) { + px[j] = round(sin(k) * size + x); + py[j] = round(cos(k) * size + y); + k += M_PI/2; + } + + /* Draw the square. */ + for (int j = 0; j < 4; j++) + lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],color); +} diff --git a/src/lolwut.h b/src/lolwut.h new file mode 100644 index 000000000..c049ac907 --- /dev/null +++ b/src/lolwut.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2018-2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* This structure represents our canvas. Drawing functions will take a pointer + * to a canvas to write to it. Later the canvas can be rendered to a string + * suitable to be printed on the screen, using unicode Braille characters. */ + +/* This represents a very simple generic canvas in order to draw stuff. + * It's up to each LOLWUT versions to translate what they draw to the + * screen, depending on the result to accomplish. */ +typedef struct lwCanvas { + int width; + int height; + char *pixels; +} lwCanvas; + +/* Drawing functions implemented inside lolwut.c. */ +lwCanvas *lwCreateCanvas(int width, int height); +void lwFreeCanvas(lwCanvas *canvas); +void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); +int lwGetPixel(lwCanvas *canvas, int x, int y); +void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color); +void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle, int color); diff --git a/src/lolwut5.c b/src/lolwut5.c index 52a98c0d7..4e9828145 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -34,17 +34,9 @@ */ #include "server.h" +#include "lolwut.h" #include -/* This structure represents our canvas. Drawing functions will take a pointer - * to a canvas to write to it. Later the canvas can be rendered to a string - * suitable to be printed on the screen, using unicode Braille characters. */ -typedef struct lwCanvas { - int width; - int height; - char *pixels; -} lwCanvas; - /* Translate a group of 8 pixels (2x4 vertical rectangle) to the corresponding * braille character. The byte should correspond to the pixels arranged as * follows, where 0 is the least significant bit, and 7 the most significant @@ -69,104 +61,6 @@ void lwTranslatePixelsGroup(int byte, char *output) { output[2] = 0x80 | (code & 0x3F); /* 10-xxxxxx */ } -/* Allocate and return a new canvas of the specified size. */ -lwCanvas *lwCreateCanvas(int width, int height) { - lwCanvas *canvas = zmalloc(sizeof(*canvas)); - canvas->width = width; - canvas->height = height; - canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,0,width*height); - return canvas; -} - -/* Free the canvas created by lwCreateCanvas(). */ -void lwFreeCanvas(lwCanvas *canvas) { - zfree(canvas->pixels); - zfree(canvas); -} - -/* Set a pixel to the specified color. Color is 0 or 1, where zero means no - * dot will be displyed, and 1 means dot will be displayed. - * Coordinates are arranged so that left-top corner is 0,0. You can write - * out of the size of the canvas without issues. */ -void lwDrawPixel(lwCanvas *canvas, int x, int y, int color) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return; - canvas->pixels[x+y*canvas->width] = color; -} - -/* Return the value of the specified pixel on the canvas. */ -int lwGetPixel(lwCanvas *canvas, int x, int y) { - if (x < 0 || x >= canvas->width || - y < 0 || y >= canvas->height) return 0; - return canvas->pixels[x+y*canvas->width]; -} - -/* Draw a line from x1,y1 to x2,y2 using the Bresenham algorithm. */ -void lwDrawLine(lwCanvas *canvas, int x1, int y1, int x2, int y2, int color) { - int dx = abs(x2-x1); - int dy = abs(y2-y1); - int sx = (x1 < x2) ? 1 : -1; - int sy = (y1 < y2) ? 1 : -1; - int err = dx-dy, e2; - - while(1) { - lwDrawPixel(canvas,x1,y1,color); - if (x1 == x2 && y1 == y2) break; - e2 = err*2; - if (e2 > -dy) { - err -= dy; - x1 += sx; - } - if (e2 < dx) { - err += dx; - y1 += sy; - } - } -} - -/* Draw a square centered at the specified x,y coordinates, with the specified - * rotation angle and size. In order to write a rotated square, we use the - * trivial fact that the parametric equation: - * - * x = sin(k) - * y = cos(k) - * - * Describes a circle for values going from 0 to 2*PI. So basically if we start - * at 45 degrees, that is k = PI/4, with the first point, and then we find - * the other three points incrementing K by PI/2 (90 degrees), we'll have the - * points of the square. In order to rotate the square, we just start with - * k = PI/4 + rotation_angle, and we are done. - * - * Of course the vanilla equations above will describe the square inside a - * circle of radius 1, so in order to draw larger squares we'll have to - * multiply the obtained coordinates, and then translate them. However this - * is much simpler than implementing the abstract concept of 2D shape and then - * performing the rotation/translation transformation, so for LOLWUT it's - * a good approach. */ -void lwDrawSquare(lwCanvas *canvas, int x, int y, float size, float angle) { - int px[4], py[4]; - - /* Adjust the desired size according to the fact that the square inscribed - * into a circle of radius 1 has the side of length SQRT(2). This way - * size becomes a simple multiplication factor we can use with our - * coordinates to magnify them. */ - size /= 1.4142135623; - size = round(size); - - /* Compute the four points. */ - float k = M_PI/4 + angle; - for (int j = 0; j < 4; j++) { - px[j] = round(sin(k) * size + x); - py[j] = round(cos(k) * size + y); - k += M_PI/2; - } - - /* Draw the square. */ - for (int j = 0; j < 4; j++) - lwDrawLine(canvas,px[j],py[j],px[(j+1)%4],py[(j+1)%4],1); -} - /* Schotter, the output of LOLWUT of Redis 5, is a computer graphic art piece * generated by Georg Nees in the 60s. It explores the relationship between * caos and order. @@ -200,7 +94,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ sx += r2*square_side/3; sy += r3*square_side/3; } - lwDrawSquare(canvas,sx,sy,square_side,angle); + lwDrawSquare(canvas,sx,sy,square_side,angle,1); } } diff --git a/src/lolwut6.c b/src/lolwut6.c new file mode 100644 index 000000000..68f09036a --- /dev/null +++ b/src/lolwut6.c @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * ---------------------------------------------------------------------------- + * + * This file implements the LOLWUT command. The command should do something + * fun and interesting, and should be replaced by a new implementation at + * each new version of Redis. + */ + +#include "server.h" +#include "lolwut.h" + +/* The LOLWUT 6 command: + * + * LOLWUT [columns] [rows] + * + * By default the command uses 80 columns, 40 squares per row + * per column. + */ +void lolwut6Command(client *c) { + long cols = 80; + long rows = 40; + + /* Parse the optional arguments if any. */ + if (c->argc > 1 && + getLongFromObjectOrReply(c,c->argv[1],&cols,NULL) != C_OK) + return; + + if (c->argc > 2 && + getLongFromObjectOrReply(c,c->argv[2],&rows,NULL) != C_OK) + return; + + /* Limits. We want LOLWUT to be always reasonably fast and cheap to execute + * so we have maximum number of columns, rows, and output resulution. */ + if (cols < 1) cols = 1; + if (cols > 1000) cols = 1000; + if (rows < 1) rows = 1; + if (rows > 1000) rows = 1000; + + /* Generate the city skyline and reply. */ + sds rendered = sdsempty(); + rendered = sdscat(rendered, + "\nDedicated to the 8 bit game developers of the past. Redis ver. "); + rendered = sdscat(rendered,REDIS_VERSION); + rendered = sdscatlen(rendered,"\n",1); + addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); + sdsfree(rendered); + // lwFreeCanvas(canvas); +} From 7dd5d37b4501216e838ddef79e6250d4a8dd3bf4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2019 19:18:45 +0200 Subject: [PATCH 158/320] LOLWUT: ability to specify VERSION option. --- src/lolwut.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index d3e8fbad5..7e2ceca78 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -50,16 +50,22 @@ void lolwutUnstableCommand(client *c) { sdsfree(rendered); } -/* LOLWUT [] */ +/* LOLWUT [VERSION ] [... version specific arguments ...] */ void lolwutCommand(client *c) { char *v = REDIS_VERSION; char verstr[64]; - if (c->argc == 2) { + if (c->argc >= 3 && !strcasecmp(c->argv[1]->ptr,"version")) { long ver; - if (getLongFromObjectOrReply(c,c->argv[1],&ver,NULL) != C_OK) return; + if (getLongFromObjectOrReply(c,c->argv[2],&ver,NULL) != C_OK) return; snprintf(verstr,sizeof(verstr),"%u.0.0",(unsigned int)ver); v = verstr; + + /* Adjust argv/argc to filter the "VERSION ..." option, since the + * specific LOLWUT version implementations don't know about it + * and expect their arguments. */ + c->argv += 2; + c->argc -= 2; } if ((v[0] == '5' && v[1] == '.' && v[2] != '9') || @@ -70,6 +76,12 @@ void lolwutCommand(client *c) { lolwut6Command(c); else lolwutUnstableCommand(c); + + /* Fix back argc/argv in case of VERSION argument. */ + if (v == verstr) { + c->argv -= 2; + c->argc += 2; + } } /* ========================== LOLWUT Canvase =============================== From dd99293067966b139e13cd6602546bd978b357b5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 6 Oct 2019 13:55:21 +0300 Subject: [PATCH 159/320] fix issues found by a static analyzer cluster.c - stack buffer memory alignment The pointer 'buf' is cast to a more strictly aligned pointer type evict.c - lazyfree_lazy_eviction, lazyfree_lazy_eviction always called defrag.c - bug in dead code server.c - casting was missing parenthesis rax.c - indentation / newline suggested an 'else if' was intended --- src/cluster.c | 32 +++++++++++++++++--------------- src/defrag.c | 2 +- src/evict.c | 7 +++---- src/rax.c | 3 ++- src/server.c | 2 +- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 1e7dcd50e..93be2aa32 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -2140,7 +2140,7 @@ void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; ssize_t nread; clusterMsg *hdr; clusterLink *link = (clusterLink*) privdata; @@ -2517,7 +2517,8 @@ void clusterBroadcastPong(int target) { * * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[sizeof(clusterMsg)], *payload; + unsigned char *payload; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; uint32_t channel_len, message_len; @@ -2537,7 +2538,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - payload = buf; + payload = (unsigned char*)buf; } else { payload = zmalloc(totlen); memcpy(payload,hdr,sizeof(*hdr)); @@ -2554,7 +2555,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { decrRefCount(channel); decrRefCount(message); - if (payload != buf) zfree(payload); + if (payload != (unsigned char*)buf) zfree(payload); } /* Send a FAIL message to all the nodes we are able to contact. @@ -2563,7 +2564,7 @@ void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { * we switch the node state to CLUSTER_NODE_FAIL and ask all the other * nodes to do the same ASAP. */ void clusterSendFail(char *nodename) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); @@ -2575,7 +2576,7 @@ void clusterSendFail(char *nodename) { * slots configuration. The node name, slots bitmap, and configEpoch info * are included. */ void clusterSendUpdate(clusterLink *link, clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; if (link == NULL) return; @@ -2583,7 +2584,7 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { memcpy(hdr->data.update.nodecfg.nodename,node->name,CLUSTER_NAMELEN); hdr->data.update.nodecfg.configEpoch = htonu64(node->configEpoch); memcpy(hdr->data.update.nodecfg.slots,node->slots,sizeof(node->slots)); - clusterSendMessage(link,buf,ntohl(hdr->totlen)); + clusterSendMessage(link,(unsigned char*)buf,ntohl(hdr->totlen)); } /* Send a MODULE message. @@ -2591,7 +2592,8 @@ void clusterSendUpdate(clusterLink *link, clusterNode *node) { * If link is NULL, then the message is broadcasted to the whole cluster. */ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len) { - unsigned char buf[sizeof(clusterMsg)], *heapbuf; + unsigned char *heapbuf; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2606,7 +2608,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, /* Try to use the local buffer if possible */ if (totlen < sizeof(buf)) { - heapbuf = buf; + heapbuf = (unsigned char*)buf; } else { heapbuf = zmalloc(totlen); memcpy(heapbuf,hdr,sizeof(*hdr)); @@ -2619,7 +2621,7 @@ void clusterSendModule(clusterLink *link, uint64_t module_id, uint8_t type, else clusterBroadcastMessage(heapbuf,totlen); - if (heapbuf != buf) zfree(heapbuf); + if (heapbuf != (unsigned char*)buf) zfree(heapbuf); } /* This function gets a cluster node ID string as target, the same way the nodes @@ -2663,7 +2665,7 @@ void clusterPropagatePublish(robj *channel, robj *message) { * Note that we send the failover request to everybody, master and slave nodes, * but only the masters are supposed to reply to our query. */ void clusterRequestFailoverAuth(void) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2679,7 +2681,7 @@ void clusterRequestFailoverAuth(void) { /* Send a FAILOVER_AUTH_ACK message to the specified node. */ void clusterSendFailoverAuth(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2687,12 +2689,12 @@ void clusterSendFailoverAuth(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAILOVER_AUTH_ACK); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Send a MFSTART message to the specified node. */ void clusterSendMFStart(clusterNode *node) { - unsigned char buf[sizeof(clusterMsg)]; + clusterMsg buf[1]; clusterMsg *hdr = (clusterMsg*) buf; uint32_t totlen; @@ -2700,7 +2702,7 @@ void clusterSendMFStart(clusterNode *node) { clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_MFSTART); totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); hdr->totlen = htonl(totlen); - clusterSendMessage(node->link,buf,totlen); + clusterSendMessage(node->link,(unsigned char*)buf,totlen); } /* Vote for the node asking for our vote if there are the conditions. */ diff --git a/src/defrag.c b/src/defrag.c index 50f6b41f2..e794c8e41 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -374,7 +374,7 @@ long activeDefragSdsListAndDict(list *l, dict *d, int dict_val_type) { if ((newele = activeDefragStringOb(ele, &defragged))) de->v.val = newele, defragged++; } else if (dict_val_type == DEFRAG_SDS_DICT_VAL_VOID_PTR) { - void *newptr, *ptr = ln->value; + void *newptr, *ptr = dictGetVal(de); if ((newptr = activeDefragAlloc(ptr))) ln->value = newptr, defragged++; } diff --git a/src/evict.c b/src/evict.c index 176f4c362..71260c040 100644 --- a/src/evict.c +++ b/src/evict.c @@ -444,6 +444,7 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * Otehrwise if we are over the memory limit, but not enough memory * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { + int keys_freed = 0; /* By default replicas should ignore maxmemory * and just be masters exact copies. */ if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; @@ -467,7 +468,7 @@ int freeMemoryIfNeeded(void) { latencyStartMonitor(latency); while (mem_freed < mem_tofree) { - int j, k, i, keys_freed = 0; + int j, k, i; static unsigned int next_db = 0; sds bestkey = NULL; int bestdbid; @@ -598,9 +599,7 @@ int freeMemoryIfNeeded(void) { mem_freed = mem_tofree; } } - } - - if (!keys_freed) { + } else { latencyEndMonitor(latency); latencyAddSampleIfNeeded("eviction-cycle",latency); goto cant_free; /* nothing to free... */ diff --git a/src/rax.c b/src/rax.c index b3c263dc4..be474b058 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1791,7 +1791,8 @@ int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key if (eq && key_len == iter->key_len) return 1; else if (lt) return iter->key_len < key_len; else if (gt) return iter->key_len > key_len; - } if (cmp > 0) { + return 0; + } else if (cmp > 0) { return gt ? 1 : 0; } else /* (cmp < 0) */ { return lt ? 1 : 0; diff --git a/src/server.c b/src/server.c index 593f98f3f..e089865f8 100644 --- a/src/server.c +++ b/src/server.c @@ -4278,7 +4278,7 @@ sds genRedisInfoString(char *section) { if (server.repl_state != REPL_STATE_CONNECTED) { info = sdscatprintf(info, "master_link_down_since_seconds:%jd\r\n", - (intmax_t)server.unixtime-server.repl_down_since); + (intmax_t)(server.unixtime-server.repl_down_since)); } info = sdscatprintf(info, "slave_priority:%d\r\n" From a8c9f29787372b15c1f4c0072393399b468bbce1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 12:35:05 +0200 Subject: [PATCH 160/320] LOLWUT: version 6 initial concept. --- src/lolwut5.c | 4 +-- src/lolwut6.c | 96 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/src/lolwut5.c b/src/lolwut5.c index 4e9828145..0200fece6 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -106,7 +106,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ * logical canvas. The actual returned string will require a terminal that is * width/2 large and height/4 tall in order to hold the whole image without * overflowing or scrolling, since each Barille character is 2x4. */ -sds lwRenderCanvas(lwCanvas *canvas) { +static sds renderCanvas(lwCanvas *canvas) { sds text = sdsempty(); for (int y = 0; y < canvas->height; y += 4) { for (int x = 0; x < canvas->width; x += 2) { @@ -166,7 +166,7 @@ void lolwut5Command(client *c) { /* Generate some computer art and reply. */ lwCanvas *canvas = lwDrawSchotter(cols,squares_per_row,squares_per_col); - sds rendered = lwRenderCanvas(canvas); + sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, "\nGeorg Nees - schotter, plotter on paper, 1968. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); diff --git a/src/lolwut6.c b/src/lolwut6.c index 68f09036a..8cc8d6e04 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -36,6 +36,94 @@ #include "server.h" #include "lolwut.h" +/* Render the canvas using the four gray levels of the standard color + * terminal: they match very well to the grayscale display of the gameboy. */ +static sds renderCanvas(lwCanvas *canvas) { + sds text = sdsempty(); + for (int y = 0; y < canvas->height; y++) { + for (int x = 0; x < canvas->width; x++) { + int color = lwGetPixel(canvas,x,y); + char *ce; /* Color escape sequence. */ + + /* Note that we set both the foreground and background color. + * This way we are able to get a more consistent result among + * different terminals implementations. */ + switch(color) { + case 0: ce = "0;30;40m"; break; /* Black */ + case 1: ce = "0;90;100m"; break; /* Gray 1 */ + case 2: ce = "0;37;47m"; break; /* Gray 2 */ + case 3: ce = "0;97;107m"; break; /* White */ + } + text = sdscatprintf(text,"\033[%s \033[0m",ce); + } + if (y != canvas->height-1) text = sdscatlen(text,"\n",1); + } + return text; +} + +/* Draw a skyscraper on the canvas, according to the parameters in the + * 'skyscraper' structure. Window colors are random and are always selected + * to be different than the color of the skyscraper itsefl. */ +struct skyscraper { + int xoff; /* X offset. */ + int width; /* Pixels width. */ + int height; /* Pixels height. */ + int windows; /* Draw windows if true. */ + int color; /* Color of the skyscraper. */ +}; + +void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { + int starty = canvas->height-1; + int endy = starty - si->height + 1; + for (int y = starty; y >= endy; y--) { + for (int x = si->xoff; x < si->xoff+si->width; x++) { + /* The roof is four pixels less wide. */ + if (y == endy && (x <= si->xoff+1 || x >= si->xoff+si->width-2)) + continue; + int color = si->color; + /* Alter the color if this is a place where we want to + * draw a window. We check that we are in the inner part of the + * skyscraper, so that windows are far from the borders. */ + if (si->windows && + x > si->xoff+1 && + x < si->xoff+si->width-2 && + y > endy+1 && + y < starty-1) + { + /* Calculate the x,y position relative to the start of + * the window area. */ + int relx = x - (si->xoff+1); + int rely = y - (endy+1); + + /* Note that we want the windows to be two pixels wide + * but just one pixel tall, because terminal "pixels" + * (characters) are not square. */ + if (relx/2 % 2 && rely % 2) { + do { + color = rand() % 4; + } while (color == si->color); + /* Except we want adjacent pixels creating the same + * window to be the same color. */ + if (relx % 2) color = lwGetPixel(canvas,x-1,y); + } + } + lwDrawPixel(canvas,x,y,color); + } + } +} + +/* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ +void generateSkyline(lwCanvas *canvas) { + struct skyscraper si = { + .xoff = 4, + .width = 13, + .height = 15, + .windows = 1, + .color = 1 + }; + generateSkyscraper(canvas, &si); +} + /* The LOLWUT 6 command: * * LOLWUT [columns] [rows] @@ -45,7 +133,7 @@ */ void lolwut6Command(client *c) { long cols = 80; - long rows = 40; + long rows = 20; /* Parse the optional arguments if any. */ if (c->argc > 1 && @@ -64,12 +152,14 @@ void lolwut6Command(client *c) { if (rows > 1000) rows = 1000; /* Generate the city skyline and reply. */ - sds rendered = sdsempty(); + lwCanvas *canvas = lwCreateCanvas(cols,rows); + generateSkyline(canvas); + sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, "\nDedicated to the 8 bit game developers of the past. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); sdsfree(rendered); - // lwFreeCanvas(canvas); + lwFreeCanvas(canvas); } From d453899c0fbf1872079aedce9753acaef52c9f43 Mon Sep 17 00:00:00 2001 From: charsyam Date: Mon, 7 Oct 2019 23:48:11 +0900 Subject: [PATCH 161/320] fix type salves to slaves --- src/replication.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index 76090a9e5..f10a91677 100644 --- a/src/replication.c +++ b/src/replication.c @@ -585,7 +585,7 @@ int startBgsaveForReplication(int mincapa) { } /* If we failed to BGSAVE, remove the slaves waiting for a full - * resynchorinization from the list of salves, inform them with + * resynchorinization from the list of slaves, inform them with * an error about what happened, close the connection ASAP. */ if (retval == C_ERR) { serverLog(LL_WARNING,"BGSAVE for replication failed"); @@ -606,7 +606,7 @@ int startBgsaveForReplication(int mincapa) { } /* If the target is socket, rdbSaveToSlavesSockets() already setup - * the salves for a full resync. Otherwise for disk target do it now.*/ + * the slaves for a full resync. Otherwise for disk target do it now.*/ if (!socket_target) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { From 58f31d22973f28a27adaa0e9182650cd78111b1e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:24:34 +0200 Subject: [PATCH 162/320] LOLWUT: version 6 initial output. May change a bit. --- src/lolwut.c | 4 ++-- src/lolwut.h | 2 +- src/lolwut5.c | 2 +- src/lolwut6.c | 38 +++++++++++++++++++++++++++++--------- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/lolwut.c b/src/lolwut.c index 7e2ceca78..0e1552ba0 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -90,12 +90,12 @@ void lolwutCommand(client *c) { * canvas implementation that can be reused. */ /* Allocate and return a new canvas of the specified size. */ -lwCanvas *lwCreateCanvas(int width, int height) { +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) { lwCanvas *canvas = zmalloc(sizeof(*canvas)); canvas->width = width; canvas->height = height; canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,0,width*height); + memset(canvas->pixels,bgcolor,width*height); return canvas; } diff --git a/src/lolwut.h b/src/lolwut.h index c049ac907..38c0de423 100644 --- a/src/lolwut.h +++ b/src/lolwut.h @@ -41,7 +41,7 @@ typedef struct lwCanvas { } lwCanvas; /* Drawing functions implemented inside lolwut.c. */ -lwCanvas *lwCreateCanvas(int width, int height); +lwCanvas *lwCreateCanvas(int width, int height, int bgcolor); void lwFreeCanvas(lwCanvas *canvas); void lwDrawPixel(lwCanvas *canvas, int x, int y, int color); int lwGetPixel(lwCanvas *canvas, int x, int y); diff --git a/src/lolwut5.c b/src/lolwut5.c index 0200fece6..5a9348800 100644 --- a/src/lolwut5.c +++ b/src/lolwut5.c @@ -74,7 +74,7 @@ lwCanvas *lwDrawSchotter(int console_cols, int squares_per_row, int squares_per_ int padding = canvas_width > 4 ? 2 : 0; float square_side = (float)(canvas_width-padding*2) / squares_per_row; int canvas_height = square_side * squares_per_col + padding*2; - lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height); + lwCanvas *canvas = lwCreateCanvas(canvas_width, canvas_height, 0); for (int y = 0; y < squares_per_col; y++) { for (int x = 0; x < squares_per_row; x++) { diff --git a/src/lolwut6.c b/src/lolwut6.c index 8cc8d6e04..44c933629 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -114,14 +114,34 @@ void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { /* Generate a skyline inspired by the parallax backgrounds of 8 bit games. */ void generateSkyline(lwCanvas *canvas) { - struct skyscraper si = { - .xoff = 4, - .width = 13, - .height = 15, - .windows = 1, - .color = 1 - }; - generateSkyscraper(canvas, &si); + struct skyscraper si; + + /* First draw the background skyscraper without windows, using the + * two different grays. */ + si.color = 1; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + si.height = canvas->height/2 + rand()%canvas->height/2; + si.windows = 0; + si.color = si.color == 1 ? 2 : 1; + generateSkyscraper(canvas, &si); + offset += si.width/2; + } + + /* Now draw the foreground skyscraper with the windows. */ + si.color = 0; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 5 + rand()%14; + if (si.width % 4) si.width += (si.width % 3); + si.height = canvas->height/2 + rand()%canvas->height/2; + si.windows = 1; + generateSkyscraper(canvas, &si); + offset += si.width+1; + } } /* The LOLWUT 6 command: @@ -152,7 +172,7 @@ void lolwut6Command(client *c) { if (rows > 1000) rows = 1000; /* Generate the city skyline and reply. */ - lwCanvas *canvas = lwCreateCanvas(cols,rows); + lwCanvas *canvas = lwCreateCanvas(cols,rows,3); generateSkyline(canvas); sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, From 0d1938be79d66932ea7bb2e1d7f7a343034d0348 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:41:29 +0200 Subject: [PATCH 163/320] LOLWUT: version 6: improve rithm of the image. --- src/lolwut6.c | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/lolwut6.c b/src/lolwut6.c index 44c933629..045f06fee 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -62,8 +62,8 @@ static sds renderCanvas(lwCanvas *canvas) { } /* Draw a skyscraper on the canvas, according to the parameters in the - * 'skyscraper' structure. Window colors are random and are always selected - * to be different than the color of the skyscraper itsefl. */ + * 'skyscraper' structure. Window colors are random and are always one + * of the two grays. */ struct skyscraper { int xoff; /* X offset. */ int width; /* Pixels width. */ @@ -100,7 +100,7 @@ void generateSkyscraper(lwCanvas *canvas, struct skyscraper *si) { * (characters) are not square. */ if (relx/2 % 2 && rely % 2) { do { - color = rand() % 4; + color = 1 + rand() % 2; } while (color == si->color); /* Except we want adjacent pixels creating the same * window to be the same color. */ @@ -117,17 +117,25 @@ void generateSkyline(lwCanvas *canvas) { struct skyscraper si; /* First draw the background skyscraper without windows, using the - * two different grays. */ - si.color = 1; - for (int offset = -10; offset < canvas->width;) { - offset += rand() % 8; - si.xoff = offset; - si.width = 10 + rand()%9; - si.height = canvas->height/2 + rand()%canvas->height/2; - si.windows = 0; - si.color = si.color == 1 ? 2 : 1; - generateSkyscraper(canvas, &si); - offset += si.width/2; + * two different grays. We use two passes to make sure that the lighter + * ones are always in the background. */ + for (int color = 2; color >= 1; color--) { + si.color = color; + for (int offset = -10; offset < canvas->width;) { + offset += rand() % 8; + si.xoff = offset; + si.width = 10 + rand()%9; + if (color == 2) + si.height = canvas->height/2 + rand()%canvas->height/2; + else + si.height = canvas->height/2 + rand()%canvas->height/3; + si.windows = 0; + generateSkyscraper(canvas, &si); + if (color == 2) + offset += si.width/2; + else + offset += si.width+1; + } } /* Now draw the foreground skyscraper with the windows. */ @@ -137,10 +145,10 @@ void generateSkyline(lwCanvas *canvas) { si.xoff = offset; si.width = 5 + rand()%14; if (si.width % 4) si.width += (si.width % 3); - si.height = canvas->height/2 + rand()%canvas->height/2; + si.height = canvas->height/3 + rand()%canvas->height/2; si.windows = 1; generateSkyscraper(canvas, &si); - offset += si.width+1; + offset += si.width+5; } } From bbed7697b2d0b4f0e72708c5acf4193bf52068b9 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 18:57:25 +0200 Subject: [PATCH 164/320] LOLWUT: version 6: give credits. --- src/lolwut6.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lolwut6.c b/src/lolwut6.c index 045f06fee..46f76cc2a 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -31,6 +31,11 @@ * This file implements the LOLWUT command. The command should do something * fun and interesting, and should be replaced by a new implementation at * each new version of Redis. + * + * Thanks to the Shhh computer art collective for the help in tuning the + * output to have a better artistic effect. Also thanks to Michele Hiki Falcone + * for sending me the original gameboy image that this LOLWUT makes + * generative. */ #include "server.h" From 0c3c2ed744d4ddde4de11e7a31414ceda5149728 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 7 Oct 2019 19:19:38 +0200 Subject: [PATCH 165/320] LOLWUT: version 6: change text message & credits. --- src/lolwut6.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/lolwut6.c b/src/lolwut6.c index 46f76cc2a..b76d80690 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -32,10 +32,11 @@ * fun and interesting, and should be replaced by a new implementation at * each new version of Redis. * + * Thanks to Michele Hiki Falcone for the original image that ispired + * the image, part of his game, Plaguemon. + * * Thanks to the Shhh computer art collective for the help in tuning the - * output to have a better artistic effect. Also thanks to Michele Hiki Falcone - * for sending me the original gameboy image that this LOLWUT makes - * generative. + * output to have a better artistic effect. */ #include "server.h" @@ -189,7 +190,8 @@ void lolwut6Command(client *c) { generateSkyline(canvas); sds rendered = renderCanvas(canvas); rendered = sdscat(rendered, - "\nDedicated to the 8 bit game developers of the past. Redis ver. "); + "\nDedicated to the 8 bit game developers of past and present.\n" + "Original 8 bit image from Plaguemon by hikikomori. Redis ver. "); rendered = sdscat(rendered,REDIS_VERSION); rendered = sdscatlen(rendered,"\n",1); addReplyVerbatim(c,rendered,sdslen(rendered),"txt"); From 3e9241c5418848f0e96b08403e4a43a322a62917 Mon Sep 17 00:00:00 2001 From: Jamison Judge Date: Mon, 7 Oct 2019 11:01:01 -0700 Subject: [PATCH 166/320] stream.h: fix typo --- src/stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stream.h b/src/stream.h index 8ae90ce77..1163b3527 100644 --- a/src/stream.h +++ b/src/stream.h @@ -88,7 +88,7 @@ typedef struct streamNACK { /* Stream propagation informations, passed to functions in order to propagate * XCLAIM commands to AOF and slaves. */ -typedef struct sreamPropInfo { +typedef struct streamPropInfo { robj *keyname; robj *groupname; } streamPropInfo; From 10ffeb03e4908b46c34f72d6b526ffa0834fd3f8 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Thu, 12 Sep 2019 10:56:54 +0300 Subject: [PATCH 167/320] TLS: Connections refactoring and TLS support. * Introduce a connection abstraction layer for all socket operations and integrate it across the code base. * Provide an optional TLS connections implementation based on OpenSSL. * Pull a newer version of hiredis with TLS support. * Tests, redis-cli updates for TLS support. --- TLS.md | 133 ++++ deps/Makefile | 6 +- deps/hiredis/.gitignore | 1 + deps/hiredis/.travis.yml | 74 +- deps/hiredis/CHANGELOG.md | 15 +- deps/hiredis/CMakeLists.txt | 90 +++ deps/hiredis/Makefile | 106 ++- deps/hiredis/README.md | 3 +- deps/hiredis/adapters/libevent.h | 112 ++- deps/hiredis/appveyor.yml | 7 +- deps/hiredis/async.c | 176 +++-- deps/hiredis/async.h | 8 + deps/hiredis/async_private.h | 72 ++ deps/hiredis/examples/CMakeLists.txt | 46 ++ deps/hiredis/examples/example-libevent-ssl.c | 73 ++ deps/hiredis/examples/example-libevent.c | 15 +- deps/hiredis/examples/example-ssl.c | 97 +++ deps/hiredis/examples/example.c | 17 +- deps/hiredis/hiredis.c | 223 +++--- deps/hiredis/hiredis.h | 101 ++- deps/hiredis/hiredis.pc.in | 11 + deps/hiredis/hiredis_ssl.h | 53 ++ deps/hiredis/hiredis_ssl.pc.in | 12 + deps/hiredis/net.c | 122 +++- deps/hiredis/net.h | 4 + deps/hiredis/read.c | 14 +- deps/hiredis/read.h | 3 +- deps/hiredis/sds.c | 2 +- deps/hiredis/sds.h | 31 +- deps/hiredis/sockcompat.c | 248 +++++++ deps/hiredis/sockcompat.h | 91 +++ deps/hiredis/ssl.c | 448 ++++++++++++ deps/hiredis/test.c | 93 ++- deps/hiredis/test.sh | 70 ++ deps/hiredis/win32.h | 18 +- redis.conf | 44 ++ src/Makefile | 8 +- src/anet.c | 22 +- src/anet.h | 12 +- src/aof.c | 2 +- src/cluster.c | 235 +++--- src/cluster.h | 2 +- src/config.c | 83 ++- src/connection.c | 383 ++++++++++ src/connection.h | 211 ++++++ src/connhelpers.h | 60 ++ src/debug.c | 5 +- src/module.c | 8 +- src/networking.c | 244 ++++--- src/rdb.c | 40 +- src/redis-cli.c | 62 +- src/replication.c | 198 +++--- src/rio.c | 144 ++-- src/rio.h | 23 +- src/scripting.c | 22 +- src/sentinel.c | 32 +- src/server.c | 35 +- src/server.h | 32 +- src/tls.c | 669 ++++++++++++++++++ tests/cluster/run.tcl | 1 + tests/cluster/tests/04-resharding.tcl | 2 + .../cluster/tests/12-replica-migration-2.tcl | 4 + tests/helpers/bg_block_op.tcl | 8 +- tests/helpers/bg_complex_data.tcl | 8 +- tests/helpers/gen_write_load.tcl | 8 +- tests/instances.tcl | 26 +- tests/integration/aof-race.tcl | 7 +- tests/integration/aof.tcl | 12 +- tests/integration/block-repl.tcl | 10 +- tests/integration/psync2-reg.tcl | 3 +- tests/integration/redis-cli.tcl | 9 +- tests/integration/replication.tcl | 14 + tests/sentinel/tests/07-down-conditions.tcl | 3 +- tests/support/cli.tcl | 19 + tests/support/cluster.tcl | 4 +- tests/support/redis.tcl | 20 +- tests/support/server.tcl | 32 +- tests/support/util.tcl | 4 +- tests/test_helper.tcl | 23 +- tests/unit/limits.tcl | 7 +- tests/unit/other.tcl | 6 +- tests/unit/protocol.tcl | 6 +- tests/unit/tls.tcl | 25 + tests/unit/wait.tcl | 5 +- utils/gen-test-certs.sh | 23 + 85 files changed, 4625 insertions(+), 835 deletions(-) create mode 100644 TLS.md create mode 100644 deps/hiredis/CMakeLists.txt create mode 100644 deps/hiredis/async_private.h create mode 100644 deps/hiredis/examples/CMakeLists.txt create mode 100644 deps/hiredis/examples/example-libevent-ssl.c create mode 100644 deps/hiredis/examples/example-ssl.c create mode 100644 deps/hiredis/hiredis.pc.in create mode 100644 deps/hiredis/hiredis_ssl.h create mode 100644 deps/hiredis/hiredis_ssl.pc.in create mode 100644 deps/hiredis/sockcompat.c create mode 100644 deps/hiredis/sockcompat.h create mode 100644 deps/hiredis/ssl.c create mode 100755 deps/hiredis/test.sh create mode 100644 src/connection.c create mode 100644 src/connection.h create mode 100644 src/connhelpers.h create mode 100644 src/tls.c create mode 100644 tests/support/cli.tcl create mode 100644 tests/unit/tls.tcl create mode 100755 utils/gen-test-certs.sh diff --git a/TLS.md b/TLS.md new file mode 100644 index 000000000..ee24a8df5 --- /dev/null +++ b/TLS.md @@ -0,0 +1,133 @@ +TLS Support -- Work In Progress +=============================== + +This is a brief note to capture current thoughts/ideas and track pending action +items. + +Getting Started +--------------- + +### Building + +To build with TLS support you'll need OpenSSL development libraries (e.g. +libssl-dev on Debian/Ubuntu). + +Run `make BUILD_TLS=yes`. + +### Tests + +To run Redis test suite with TLS, you'll need TLS support for TCL (i.e. +`tcl-tls` package on Debian/Ubuntu). + +1. Run `./utils/gen-test-certs.sh` to generate a root CA and a server + certificate. + +2. Run `./runtest --tls` or `./runtest-cluster --tls` to run Redis and Redis + Cluster tests in TLS mode. + +### Running manually + +To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was +invoked so sample certificates/keys are available): + + ./src/redis-server --tls-port 6379 --port 0 \ + --tls-cert-file ./tests/tls/redis.crt \ + --tls-key-file ./tests/tls/redis.key \ + --tls-ca-cert-file ./tests/tls/ca.crt + +To connect to this Redis server with `redis-cli`: + + ./src/redis-cli --tls \ + --cert ./tests/tls/redis.crt \ + --key ./tests/tls/redis.key \ + --cacert ./tests/tls/ca.crt + +This will disable TCP and enable TLS on port 6379. It's also possible to have +both TCP and TLS available, but you'll need to assign different ports. + +To make a Replica connect to the master using TLS, use `--tls-replication yes`, +and to make Redis Cluster use TLS across nodes use `--tls-cluster yes`. + +**NOTE: This is still very much work in progress and some configuration is still +missing or may change.** + +Connections +----------- + +Connection abstraction API is mostly done and seems to hold well for hiding +implementation details between TLS and TCP. + +1. Still need to implement the equivalent of AE_BARRIER. Because TLS + socket-level read/write events don't correspond to logical operations, this + should probably be done at the Read/Write handler level. + +2. Multi-threading I/O is not supported. The main issue to address is the need + to manipulate AE based on OpenSSL return codes. We can either propagate this + out of the thread, or explore ways of further optimizing MT I/O by having + event loops that live inside the thread and borrow connections in/out. + +3. Finish cleaning up the implementation. Make sure all error cases are handled + and reflected into connection state, connection state validated before + certain operations, etc. + - Clean (non-errno) interface to report would-block. + - Consistent error reporting. + +4. Sync IO for TLS is currently implemented in a hackish way, i.e. making the + socket blocking and configuring socket-level timeout. This means the timeout + value may not be so accurate, and there would be a lot of syscall overhead. + However I believe that getting rid of syncio completely in favor of pure + async work is probably a better move than trying to fix that. For replication + it would probably not be so hard. For cluster keys migration it might be more + difficult, but there are probably other good reasons to improve that part + anyway. + +5. A mechanism to re-trigger read callbacks for connections with unread buffers + (the case of reading partial TLS frames): + + a) Before sleep should iterate connections looking for those with a read handler, + SSL_pending() != 0 and no read event. + b) If found, trigger read handler for these conns. + c) After iteration if this state persists, epoll should be called in a way + that won't block so the process continues and this behave the same as a + level trigerred epoll. + +Replication +----------- + +Diskless master replication is broken, until child/parent connection proxying is +implemented. + + +TLS Features +------------ + +1. Add metrics to INFO. +2. Add certificate authentication configuration (i.e. option to skip client +auth, master auth, etc.). +3. Add TLS cipher configuration options. +4. [Optional] Add session caching support. Check if/how it's handled by clients + to assess how useful/important it is. + + +redis-benchmark +--------------- + +The current implementation is a mix of using hiredis for parsing and basic +networking (establishing connections), but directly manipulating sockets for +most actions. + +This will need to be cleaned up for proper TLS support. The best approach is +probably to migrate to hiredis async mode. + + +Others +------ + +Consider the implications of allowing TLS to be configured on a separate port, +making Redis listening on multiple ports. + +This impacts many things, like +1. Startup banner port notification +2. Proctitle +3. How slaves announce themselves +4. Cluster bus port calculation diff --git a/deps/Makefile b/deps/Makefile index eb35c1e1f..700867f3b 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -41,9 +41,13 @@ distclean: .PHONY: distclean +ifeq ($(BUILD_TLS),yes) + HIREDIS_MAKE_FLAGS = USE_SSL=1 +endif + hiredis: .make-prerequisites @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) - cd hiredis && $(MAKE) static + cd hiredis && $(MAKE) static $(HIREDIS_MAKE_FLAGS) .PHONY: hiredis diff --git a/deps/hiredis/.gitignore b/deps/hiredis/.gitignore index c44b5c537..8e50b5434 100644 --- a/deps/hiredis/.gitignore +++ b/deps/hiredis/.gitignore @@ -5,3 +5,4 @@ /*.dylib /*.a /*.pc +*.dSYM diff --git a/deps/hiredis/.travis.yml b/deps/hiredis/.travis.yml index faf2ce684..dd8e0e73d 100644 --- a/deps/hiredis/.travis.yml +++ b/deps/hiredis/.travis.yml @@ -26,20 +26,72 @@ addons: - libc6-dev-i386 - libc6-dbg:i386 - gcc-multilib + - g++-multilib - valgrind env: - - CFLAGS="-Werror" - - PRE="valgrind --track-origins=yes --leak-check=full" - - TARGET="32bit" TARGET_VARS="32bit-vars" CFLAGS="-Werror" - - TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" + - BITS="32" + - BITS="64" + +script: + - EXTRA_CMAKE_OPTS="-DENABLE_EXAMPLES:BOOL=ON -DHIREDIS_SSL:BOOL=ON"; + if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + else + TEST_PREFIX="valgrind --track-origins=yes --leak-check=full"; + if [ "$BITS" == "32" ]; then + CFLAGS="-m32 -Werror"; + CXXFLAGS="-m32 -Werror"; + LDFLAGS="-m32"; + EXTRA_CMAKE_OPTS=; + else + CFLAGS="-Werror"; + CXXFLAGS="-Werror"; + fi; + fi; + export CFLAGS CXXFLAGS LDFLAGS TEST_PREFIX EXTRA_CMAKE_OPTS + - mkdir build/ && cd build/ + - cmake .. ${EXTRA_CMAKE_OPTS} + - make VERBOSE=1 + - ctest -V matrix: - exclude: - - os: osx - env: PRE="valgrind --track-origins=yes --leak-check=full" + include: + # Windows MinGW cross compile on Linux + - os: linux + dist: xenial + compiler: mingw + addons: + apt: + packages: + - ninja-build + - gcc-mingw-w64-x86-64 + - g++-mingw-w64-x86-64 + script: + - mkdir build && cd build + - CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_WITH_INSTALL_RPATH=on + - ninja -v - - os: osx - env: TARGET="32bit" TARGET_VARS="32bit-vars" PRE="valgrind --track-origins=yes --leak-check=full" - -script: make $TARGET CFLAGS="$CFLAGS" && make check PRE="$PRE" && make $TARGET_VARS hiredis-example + # Windows MSVC 2017 + - os: windows + compiler: msvc + env: + - MATRIX_EVAL="CC=cl.exe && CXX=cl.exe" + before_install: + - eval "${MATRIX_EVAL}" + install: + - choco install ninja + script: + - mkdir build && cd build + - cmd.exe /C '"C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 && + cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release && + ninja -v' + - ctest -V diff --git a/deps/hiredis/CHANGELOG.md b/deps/hiredis/CHANGELOG.md index a7fe3ac11..d1d37e515 100644 --- a/deps/hiredis/CHANGELOG.md +++ b/deps/hiredis/CHANGELOG.md @@ -12,6 +12,16 @@ compare to other values, casting might be necessary or can be removed, if casting was applied before. +### 0.x.x (unreleased) +**BREAKING CHANGES**: + +* Change `redisReply.len` to `size_t`, as it denotes the the size of a string + +User code should compare this to `size_t` values as well. +If it was used to compare to other values, casting might be necessary or can be removed, if casting was applied before. + +* `redisReplyObjectFunctions.createArray` now takes `size_t` for its length parameter. + ### 0.14.0 (2018-09-25) * Make string2ll static to fix conflict with Redis (Tom Lee [c3188b]) @@ -50,8 +60,9 @@ * Import latest upstream sds. This breaks applications that are linked against the old hiredis v0.13 * Fix warnings, when compiled with -Wshadow * Make hiredis compile in Cygwin on Windows, now CI-tested - -**BREAKING CHANGES**: +* Bulk and multi-bulk lengths less than -1 or greater than `LLONG_MAX` are now + protocol errors. This is consistent with the RESP specification. On 32-bit + platforms, the upper bound is lowered to `SIZE_MAX`. * Remove backwards compatibility macro's diff --git a/deps/hiredis/CMakeLists.txt b/deps/hiredis/CMakeLists.txt new file mode 100644 index 000000000..9e78894f3 --- /dev/null +++ b/deps/hiredis/CMakeLists.txt @@ -0,0 +1,90 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.4.0) +INCLUDE(GNUInstallDirs) +PROJECT(hiredis) + +OPTION(ENABLE_SSL "Build hiredis_ssl for SSL support" OFF) + +MACRO(getVersionBit name) + SET(VERSION_REGEX "^#define ${name} (.+)$") + FILE(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/hiredis.h" + VERSION_BIT REGEX ${VERSION_REGEX}) + STRING(REGEX REPLACE ${VERSION_REGEX} "\\1" ${name} "${VERSION_BIT}") +ENDMACRO(getVersionBit) + +getVersionBit(HIREDIS_MAJOR) +getVersionBit(HIREDIS_MINOR) +getVersionBit(HIREDIS_PATCH) +getVersionBit(HIREDIS_SONAME) +SET(VERSION "${HIREDIS_MAJOR}.${HIREDIS_MINOR}.${HIREDIS_PATCH}") +MESSAGE("Detected version: ${VERSION}") + +PROJECT(hiredis VERSION "${VERSION}") + +SET(ENABLE_EXAMPLES OFF CACHE BOOL "Enable building hiredis examples") + +ADD_LIBRARY(hiredis SHARED + async.c + dict.c + hiredis.c + net.c + read.c + sds.c + sockcompat.c) + +SET_TARGET_PROPERTIES(hiredis + PROPERTIES + VERSION "${HIREDIS_SONAME}") +IF(WIN32 OR MINGW) + TARGET_LINK_LIBRARIES(hiredis PRIVATE ws2_32) +ENDIF() +TARGET_INCLUDE_DIRECTORIES(hiredis PUBLIC .) + +CONFIGURE_FILE(hiredis.pc.in hiredis.pc @ONLY) + +INSTALL(TARGETS hiredis + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + +INSTALL(FILES hiredis.h read.h sds.h async.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(DIRECTORY adapters + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + +INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +IF(ENABLE_SSL) + IF (NOT OPENSSL_ROOT_DIR) + IF (APPLE) + SET(OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + ENDIF() + ENDIF() + FIND_PACKAGE(OpenSSL REQUIRED) + ADD_LIBRARY(hiredis_ssl SHARED + ssl.c) + TARGET_INCLUDE_DIRECTORIES(hiredis_ssl PRIVATE "${OPENSSL_INCLUDE_DIR}") + TARGET_LINK_LIBRARIES(hiredis_ssl PRIVATE ${OPENSSL_LIBRARIES}) + CONFIGURE_FILE(hiredis_ssl.pc.in hiredis_ssl.pc @ONLY) + + INSTALL(TARGETS hiredis_ssl + DESTINATION "${CMAKE_INSTALL_LIBDIR}") + + INSTALL(FILES hiredis_ssl.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/hiredis) + + INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/hiredis_ssl.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) +ENDIF() + +IF(NOT (WIN32 OR MINGW)) + ENABLE_TESTING() + ADD_EXECUTABLE(hiredis-test test.c) + TARGET_LINK_LIBRARIES(hiredis-test hiredis) + ADD_TEST(NAME hiredis-test + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/test.sh) +ENDIF() + +# Add examples +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF(ENABLE_EXAMPLES) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index 06ca99468..25ac15464 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -3,11 +3,17 @@ # Copyright (C) 2010-2011 Pieter Noordhuis # This file is released under the BSD license, see the COPYING file -OBJ=net.o hiredis.o sds.o async.o read.o +OBJ=net.o hiredis.o sds.o async.o read.o sockcompat.o +SSL_OBJ=ssl.o EXAMPLES=hiredis-example hiredis-example-libevent hiredis-example-libev hiredis-example-glib +ifeq ($(USE_SSL),1) +EXAMPLES+=hiredis-example-ssl hiredis-example-libevent-ssl +endif TESTS=hiredis-test LIBNAME=libhiredis +SSL_LIBNAME=libhiredis_ssl PKGCONFNAME=hiredis.pc +SSL_PKGCONFNAME=hiredis_ssl.pc HIREDIS_MAJOR=$(shell grep HIREDIS_MAJOR hiredis.h | awk '{print $$3}') HIREDIS_MINOR=$(shell grep HIREDIS_MINOR hiredis.h | awk '{print $$3}') @@ -39,7 +45,7 @@ export REDIS_TEST_CONFIG CC:=$(shell sh -c 'type $${CC%% *} >/dev/null 2>/dev/null && echo $(CC) || echo gcc') CXX:=$(shell sh -c 'type $${CXX%% *} >/dev/null 2>/dev/null && echo $(CXX) || echo g++') OPTIMIZATION?=-O3 -WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings +WARNINGS=-Wall -W -Wstrict-prototypes -Wwrite-strings -Wno-missing-field-initializers DEBUG_FLAGS?= -g -ggdb REAL_CFLAGS=$(OPTIMIZATION) -fPIC $(CPPFLAGS) $(CFLAGS) $(WARNINGS) $(DEBUG_FLAGS) REAL_LDFLAGS=$(LDFLAGS) @@ -49,12 +55,30 @@ STLIBSUFFIX=a DYLIB_MINOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_SONAME) DYLIB_MAJOR_NAME=$(LIBNAME).$(DYLIBSUFFIX).$(HIREDIS_MAJOR) DYLIBNAME=$(LIBNAME).$(DYLIBSUFFIX) -DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) -o $(DYLIBNAME) $(LDFLAGS) +SSL_DYLIBNAME=$(SSL_LIBNAME).$(DYLIBSUFFIX) +DYLIB_MAKE_CMD=$(CC) -shared -Wl,-soname,$(DYLIB_MINOR_NAME) STLIBNAME=$(LIBNAME).$(STLIBSUFFIX) -STLIB_MAKE_CMD=ar rcs $(STLIBNAME) +SSL_STLIBNAME=$(SSL_LIBNAME).$(STLIBSUFFIX) +STLIB_MAKE_CMD=$(AR) rcs # Platform-specific overrides uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') + +USE_SSL?=0 + +# This is required for test.c only +ifeq ($(USE_SSL),1) + CFLAGS+=-DHIREDIS_TEST_SSL +endif + +ifeq ($(uname_S),Linux) + SSL_LDFLAGS=-lssl -lcrypto +else + OPENSSL_PREFIX?=/usr/local/opt/openssl + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +endif + ifeq ($(uname_S),SunOS) REAL_LDFLAGS+= -ldl -lnsl -lsocket DYLIB_MAKE_CMD=$(CC) -G -o $(DYLIBNAME) -h $(DYLIB_MINOR_NAME) $(LDFLAGS) @@ -66,40 +90,61 @@ ifeq ($(uname_S),Darwin) endif all: $(DYLIBNAME) $(STLIBNAME) hiredis-test $(PKGCONFNAME) +ifeq ($(USE_SSL),1) +all: $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(SSL_PKGCONFNAME) +endif # Deps (use make dep to generate this) async.o: async.c fmacros.h async.h hiredis.h read.h sds.h net.h dict.c dict.h dict.o: dict.c fmacros.h dict.h -hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h -net.o: net.c fmacros.h net.h hiredis.h read.h sds.h +hiredis.o: hiredis.c fmacros.h hiredis.h read.h sds.h net.h win32.h +net.o: net.c fmacros.h net.h hiredis.h read.h sds.h sockcompat.h win32.h read.o: read.c fmacros.h read.h sds.h sds.o: sds.c sds.h +sockcompat.o: sockcompat.c sockcompat.h +ssl.o: ssl.c hiredis.h test.o: test.c fmacros.h hiredis.h read.h sds.h $(DYLIBNAME): $(OBJ) - $(DYLIB_MAKE_CMD) $(OBJ) + $(DYLIB_MAKE_CMD) -o $(DYLIBNAME) $(OBJ) $(REAL_LDFLAGS) $(STLIBNAME): $(OBJ) - $(STLIB_MAKE_CMD) $(OBJ) + $(STLIB_MAKE_CMD) $(STLIBNAME) $(OBJ) + +$(SSL_DYLIBNAME): $(SSL_OBJ) + $(DYLIB_MAKE_CMD) -o $(SSL_DYLIBNAME) $(SSL_OBJ) $(REAL_LDFLAGS) $(SSL_LDFLAGS) + +$(SSL_STLIBNAME): $(SSL_OBJ) + $(STLIB_MAKE_CMD) $(SSL_STLIBNAME) $(SSL_OBJ) dynamic: $(DYLIBNAME) static: $(STLIBNAME) +ifeq ($(USE_SSL),1) +dynamic: $(SSL_DYLIBNAME) +static: $(SSL_STLIBNAME) +endif # Binaries: hiredis-example-libevent: examples/example-libevent.c adapters/libevent.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -levent $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-libevent-ssl: examples/example-libevent-ssl.c adapters/libevent.h $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -levent $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) hiredis-example-libev: examples/example-libev.c adapters/libev.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -lev $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -lev $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-glib: examples/example-glib.c adapters/glib.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(shell pkg-config --cflags --libs glib-2.0) $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-ivykis: examples/example-ivykis.c adapters/ivykis.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -livykis $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -livykis $(STLIBNAME) $(REAL_LDFLAGS) hiredis-example-macosx: examples/example-macosx.c adapters/macosx.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< -framework CoreFoundation $(STLIBNAME) $(REAL_LDFLAGS) + +hiredis-example-ssl: examples/example-ssl.c $(STLIBNAME) $(SSL_STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(SSL_STLIBNAME) $(REAL_LDFLAGS) $(SSL_LDFLAGS) ifndef AE_DIR hiredis-example-ae: @@ -116,7 +161,7 @@ hiredis-example-libuv: @false else hiredis-example-libuv: examples/example-libuv.c adapters/libuv.h $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. -I$(LIBUV_DIR)/include $< $(LIBUV_DIR)/.libs/libuv.a -lpthread -lrt $(STLIBNAME) $(REAL_LDFLAGS) endif ifeq ($(and $(QT_MOC),$(QT_INCLUDE_DIR),$(QT_LIBRARY_DIR)),) @@ -133,32 +178,33 @@ hiredis-example-qt: examples/example-qt.cpp adapters/qt.h $(STLIBNAME) endif hiredis-example: examples/example.c $(STLIBNAME) - $(CC) -o examples/$@ $(REAL_CFLAGS) $(REAL_LDFLAGS) -I. $< $(STLIBNAME) + $(CC) -o examples/$@ $(REAL_CFLAGS) -I. $< $(STLIBNAME) $(REAL_LDFLAGS) examples: $(EXAMPLES) -hiredis-test: test.o $(STLIBNAME) +TEST_LIBS = $(STLIBNAME) +ifeq ($(USE_SSL),1) + TEST_LIBS += $(SSL_STLIBNAME) -lssl -lcrypto -lpthread +endif +hiredis-test: test.o $(TEST_LIBS) hiredis-%: %.o $(STLIBNAME) - $(CC) $(REAL_CFLAGS) -o $@ $(REAL_LDFLAGS) $< $(STLIBNAME) + $(CC) $(REAL_CFLAGS) -o $@ $< $(TEST_LIBS) $(REAL_LDFLAGS) test: hiredis-test ./hiredis-test check: hiredis-test - @echo "$$REDIS_TEST_CONFIG" | $(REDIS_SERVER) - - $(PRE) ./hiredis-test -h 127.0.0.1 -p $(REDIS_PORT) -s /tmp/hiredis-test-redis.sock || \ - ( kill `cat /tmp/hiredis-test-redis.pid` && false ) - kill `cat /tmp/hiredis-test-redis.pid` + TEST_SSL=$(USE_SSL) ./test.sh .c.o: $(CC) -std=c99 -pedantic -c $(REAL_CFLAGS) $< clean: - rm -rf $(DYLIBNAME) $(STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov + rm -rf $(DYLIBNAME) $(STLIBNAME) $(SSL_DYLIBNAME) $(SSL_STLIBNAME) $(TESTS) $(PKGCONFNAME) examples/hiredis-example* *.o *.gcda *.gcno *.gcov dep: - $(CC) -MM *.c + $(CC) $(CPPFLAGS) $(CFLAGS) -MM *.c INSTALL?= cp -pPR @@ -175,6 +221,20 @@ $(PKGCONFNAME): hiredis.h @echo Libs: -L\$${libdir} -lhiredis >> $@ @echo Cflags: -I\$${includedir} -D_FILE_OFFSET_BITS=64 >> $@ +$(SSL_PKGCONFNAME): hiredis.h + @echo "Generating $@ for pkgconfig..." + @echo prefix=$(PREFIX) > $@ + @echo exec_prefix=\$${prefix} >> $@ + @echo libdir=$(PREFIX)/$(LIBRARY_PATH) >> $@ + @echo includedir=$(PREFIX)/$(INCLUDE_PATH) >> $@ + @echo >> $@ + @echo Name: hiredis_ssl >> $@ + @echo Description: SSL Support for hiredis. >> $@ + @echo Version: $(HIREDIS_MAJOR).$(HIREDIS_MINOR).$(HIREDIS_PATCH) >> $@ + @echo Requires: hiredis >> $@ + @echo Libs: -L\$${libdir} -lhiredis_ssl >> $@ + @echo Libs.private: -lssl -lcrypto >> $@ + install: $(DYLIBNAME) $(STLIBNAME) $(PKGCONFNAME) mkdir -p $(INSTALL_INCLUDE_PATH) $(INSTALL_INCLUDE_PATH)/adapters $(INSTALL_LIBRARY_PATH) $(INSTALL) hiredis.h async.h read.h sds.h $(INSTALL_INCLUDE_PATH) diff --git a/deps/hiredis/README.md b/deps/hiredis/README.md index 01223ea59..c0b432f07 100644 --- a/deps/hiredis/README.md +++ b/deps/hiredis/README.md @@ -286,6 +286,7 @@ return `REDIS_ERR`. The function to set the disconnect callback has the followin ```c int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); ``` +`ac->data` may be used to pass user data to this callback, the same can be done for redisConnectCallback. ### Sending commands and their callbacks In an asynchronous context, commands are automatically pipelined due to the nature of an event loop. @@ -406,6 +407,6 @@ as soon as possible in order to prevent allocation of useless memory. ## AUTHORS Hiredis was written by Salvatore Sanfilippo (antirez at gmail) and -Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. +Pieter Noordhuis (pcnoordhuis at gmail) and is released under the BSD license. Hiredis is currently maintained by Matt Stancliff (matt at genges dot com) and Jan-Erik Rediger (janerik at fnordig dot com) diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 7d2bef180..a4952776c 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -34,48 +34,113 @@ #include "../hiredis.h" #include "../async.h" +#define REDIS_LIBEVENT_DELETED 0x01 +#define REDIS_LIBEVENT_ENTERED 0x02 + typedef struct redisLibeventEvents { redisAsyncContext *context; - struct event *rev, *wev; + struct event *ev; + struct event_base *base; + struct timeval tv; + short flags; + short state; } redisLibeventEvents; -static void redisLibeventReadEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); - redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleRead(e->context); +static void redisLibeventDestroy(redisLibeventEvents *e) { + free(e); } -static void redisLibeventWriteEvent(int fd, short event, void *arg) { - ((void)fd); ((void)event); +static void redisLibeventHandler(int fd, short event, void *arg) { + ((void)fd); redisLibeventEvents *e = (redisLibeventEvents*)arg; - redisAsyncHandleWrite(e->context); + e->state |= REDIS_LIBEVENT_ENTERED; + + #define CHECK_DELETED() if (e->state & REDIS_LIBEVENT_DELETED) {\ + redisLibeventDestroy(e);\ + return; \ + } + + if ((event & EV_TIMEOUT) && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleTimeout(e->context); + CHECK_DELETED(); + } + + if ((event & EV_READ) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleRead(e->context); + CHECK_DELETED(); + } + + if ((event & EV_WRITE) && e->context && (e->state & REDIS_LIBEVENT_DELETED) == 0) { + redisAsyncHandleWrite(e->context); + CHECK_DELETED(); + } + + e->state &= ~REDIS_LIBEVENT_ENTERED; + #undef CHECK_DELETED +} + +static void redisLibeventUpdate(void *privdata, short flag, int isRemove) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + const struct timeval *tv = e->tv.tv_sec || e->tv.tv_usec ? &e->tv : NULL; + + if (isRemove) { + if ((e->flags & flag) == 0) { + return; + } else { + e->flags &= ~flag; + } + } else { + if (e->flags & flag) { + return; + } else { + e->flags |= flag; + } + } + + event_del(e->ev); + event_assign(e->ev, e->base, e->context->c.fd, e->flags | EV_PERSIST, + redisLibeventHandler, privdata); + event_add(e->ev, tv); } static void redisLibeventAddRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->rev,NULL); + redisLibeventUpdate(privdata, EV_READ, 0); } static void redisLibeventDelRead(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->rev); + redisLibeventUpdate(privdata, EV_READ, 1); } static void redisLibeventAddWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_add(e->wev,NULL); + redisLibeventUpdate(privdata, EV_WRITE, 0); } static void redisLibeventDelWrite(void *privdata) { - redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_del(e->wev); + redisLibeventUpdate(privdata, EV_WRITE, 1); } static void redisLibeventCleanup(void *privdata) { redisLibeventEvents *e = (redisLibeventEvents*)privdata; - event_free(e->rev); - event_free(e->wev); - free(e); + if (!e) { + return; + } + event_del(e->ev); + event_free(e->ev); + e->ev = NULL; + + if (e->state & REDIS_LIBEVENT_ENTERED) { + e->state |= REDIS_LIBEVENT_DELETED; + } else { + redisLibeventDestroy(e); + } +} + +static void redisLibeventSetTimeout(void *privdata, struct timeval tv) { + redisLibeventEvents *e = (redisLibeventEvents *)privdata; + short flags = e->flags; + e->flags = 0; + e->tv = tv; + redisLibeventUpdate(e, flags, 0); } static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { @@ -87,7 +152,7 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { return REDIS_ERR; /* Create container for context and r/w events */ - e = (redisLibeventEvents*)malloc(sizeof(*e)); + e = (redisLibeventEvents*)calloc(1, sizeof(*e)); e->context = ac; /* Register functions to start/stop listening for events */ @@ -96,13 +161,12 @@ static int redisLibeventAttach(redisAsyncContext *ac, struct event_base *base) { ac->ev.addWrite = redisLibeventAddWrite; ac->ev.delWrite = redisLibeventDelWrite; ac->ev.cleanup = redisLibeventCleanup; + ac->ev.scheduleTimer = redisLibeventSetTimeout; ac->ev.data = e; /* Initialize and install read/write events */ - e->rev = event_new(base, c->fd, EV_READ, redisLibeventReadEvent, e); - e->wev = event_new(base, c->fd, EV_WRITE, redisLibeventWriteEvent, e); - event_add(e->rev, NULL); - event_add(e->wev, NULL); + e->ev = event_new(base, c->fd, EV_READ | EV_WRITE, redisLibeventHandler, e); + e->base = base; return REDIS_OK; } #endif diff --git a/deps/hiredis/appveyor.yml b/deps/hiredis/appveyor.yml index 819efbd58..5b43fdbeb 100644 --- a/deps/hiredis/appveyor.yml +++ b/deps/hiredis/appveyor.yml @@ -5,8 +5,9 @@ environment: CC: gcc - CYG_BASH: C:\cygwin\bin\bash CC: gcc - TARGET: 32bit - TARGET_VARS: 32bit-vars + CFLAGS: -m32 + CXXFLAGS: -m32 + LDFLAGS: -m32 clone_depth: 1 @@ -20,4 +21,4 @@ install: build_script: - 'echo building...' - - '%CYG_BASH% -lc "cd $APPVEYOR_BUILD_FOLDER; exec 0 #include +#ifndef _MSC_VER #include +#endif #include #include #include @@ -40,22 +42,9 @@ #include "net.h" #include "dict.c" #include "sds.h" +#include "win32.h" -#define _EL_ADD_READ(ctx) do { \ - if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_READ(ctx) do { \ - if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ - } while(0) -#define _EL_ADD_WRITE(ctx) do { \ - if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ - } while(0) -#define _EL_DEL_WRITE(ctx) do { \ - if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ - } while(0) -#define _EL_CLEANUP(ctx) do { \ - if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ - } while(0); +#include "async_private.h" /* Forward declaration of function in hiredis.c */ int __redisAppendCommand(redisContext *c, const char *cmd, size_t len); @@ -126,6 +115,7 @@ static redisAsyncContext *redisAsyncInitialize(redisContext *c) { ac->ev.addWrite = NULL; ac->ev.delWrite = NULL; ac->ev.cleanup = NULL; + ac->ev.scheduleTimer = NULL; ac->onConnect = NULL; ac->onDisconnect = NULL; @@ -150,56 +140,52 @@ static void __redisAsyncCopyError(redisAsyncContext *ac) { ac->errstr = c->errstr; } -redisAsyncContext *redisAsyncConnect(const char *ip, int port) { +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options) { + redisOptions myOptions = *options; redisContext *c; redisAsyncContext *ac; - c = redisConnectNonBlock(ip,port); - if (c == NULL) + myOptions.options |= REDIS_OPT_NONBLOCK; + c = redisConnectWithOptions(&myOptions); + if (c == NULL) { return NULL; - + } ac = redisAsyncInitialize(c); if (ac == NULL) { redisFree(c); return NULL; } - __redisAsyncCopyError(ac); return ac; } +redisAsyncContext *redisAsyncConnect(const char *ip, int port) { + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisAsyncConnectWithOptions(&options); +} + redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlock(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisConnectBindNonBlockWithReuse(ip,port,source_addr); - redisAsyncContext *ac = redisAsyncInitialize(c); - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_REUSEADDR; + options.endpoint.tcp.source_addr = source_addr; + return redisAsyncConnectWithOptions(&options); } redisAsyncContext *redisAsyncConnectUnix(const char *path) { - redisContext *c; - redisAsyncContext *ac; - - c = redisConnectUnixNonBlock(path); - if (c == NULL) - return NULL; - - ac = redisAsyncInitialize(c); - if (ac == NULL) { - redisFree(c); - return NULL; - } - - __redisAsyncCopyError(ac); - return ac; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisAsyncConnectWithOptions(&options); } int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn) { @@ -328,7 +314,7 @@ void redisAsyncFree(redisAsyncContext *ac) { } /* Helper function to make the disconnect happen and clean up. */ -static void __redisAsyncDisconnect(redisAsyncContext *ac) { +void __redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); /* Make sure error is accessible if there is any */ @@ -344,9 +330,15 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { c->flags |= REDIS_DISCONNECTING; } + /* cleanup event library on disconnect. + * this is safe to call multiple times */ + _EL_CLEANUP(ac); + /* For non-clean disconnects, __redisAsyncFree() will execute pending * callbacks with a NULL-reply. */ - __redisAsyncFree(ac); + if (!(c->flags & REDIS_NO_AUTO_FREE)) { + __redisAsyncFree(ac); + } } /* Tries to do a clean disconnect from Redis, meaning it stops new commands @@ -358,6 +350,9 @@ static void __redisAsyncDisconnect(redisAsyncContext *ac) { void redisAsyncDisconnect(redisAsyncContext *ac) { redisContext *c = &(ac->c); c->flags |= REDIS_DISCONNECTING; + + /** unset the auto-free flag here, because disconnect undoes this */ + c->flags &= ~REDIS_NO_AUTO_FREE; if (!(c->flags & REDIS_IN_CALLBACK) && ac->replies.head == NULL) __redisAsyncDisconnect(ac); } @@ -408,7 +403,7 @@ static int __redisGetSubscribeCallback(redisAsyncContext *ac, redisReply *reply, assert(reply->element[2]->type == REDIS_REPLY_INTEGER); /* Unset subscribed flag only when no pipelined pending subscribe. */ - if (reply->element[2]->integer == 0 + if (reply->element[2]->integer == 0 && dictSize(ac->sub.channels) == 0 && dictSize(ac->sub.patterns) == 0) c->flags &= ~REDIS_SUBSCRIBED; @@ -524,6 +519,18 @@ static int __redisAsyncHandleConnect(redisAsyncContext *ac) { } } +void redisAsyncRead(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (redisBufferRead(c) == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + /* Always re-schedule reads */ + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + /* This function should be called when the socket is readable. * It processes all replies that can be read and executes their callbacks. */ @@ -539,28 +546,13 @@ void redisAsyncHandleRead(redisAsyncContext *ac) { return; } - if (redisBufferRead(c) == REDIS_ERR) { - __redisAsyncDisconnect(ac); - } else { - /* Always re-schedule reads */ - _EL_ADD_READ(ac); - redisProcessCallbacks(ac); - } + c->funcs->async_read(ac); } -void redisAsyncHandleWrite(redisAsyncContext *ac) { +void redisAsyncWrite(redisAsyncContext *ac) { redisContext *c = &(ac->c); int done = 0; - if (!(c->flags & REDIS_CONNECTED)) { - /* Abort connect was not successful. */ - if (__redisAsyncHandleConnect(ac) != REDIS_OK) - return; - /* Try again later when the context is still not connected. */ - if (!(c->flags & REDIS_CONNECTED)) - return; - } - if (redisBufferWrite(c,&done) == REDIS_ERR) { __redisAsyncDisconnect(ac); } else { @@ -575,6 +567,51 @@ void redisAsyncHandleWrite(redisAsyncContext *ac) { } } +void redisAsyncHandleWrite(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + + if (!(c->flags & REDIS_CONNECTED)) { + /* Abort connect was not successful. */ + if (__redisAsyncHandleConnect(ac) != REDIS_OK) + return; + /* Try again later when the context is still not connected. */ + if (!(c->flags & REDIS_CONNECTED)) + return; + } + + c->funcs->async_write(ac); +} + +void __redisSetError(redisContext *c, int type, const char *str); + +void redisAsyncHandleTimeout(redisAsyncContext *ac) { + redisContext *c = &(ac->c); + redisCallback cb; + + if ((c->flags & REDIS_CONNECTED) && ac->replies.head == NULL) { + /* Nothing to do - just an idle timeout */ + return; + } + + if (!c->err) { + __redisSetError(c, REDIS_ERR_TIMEOUT, "Timeout"); + } + + if (!(c->flags & REDIS_CONNECTED) && ac->onConnect) { + ac->onConnect(ac, REDIS_ERR); + } + + while (__redisShiftCallback(&ac->replies, &cb) == REDIS_OK) { + __redisRunCallback(ac, &cb, NULL); + } + + /** + * TODO: Don't automatically sever the connection, + * rather, allow to ignore responses before the queue is clear + */ + __redisAsyncDisconnect(ac); +} + /* Sets a pointer to the first argument and its length starting at p. Returns * the number of bytes to skip to get to the following argument. */ static const char *nextArgument(const char *start, const char **str, size_t *len) { @@ -714,3 +751,16 @@ int redisAsyncFormattedCommand(redisAsyncContext *ac, redisCallbackFn *fn, void int status = __redisAsyncCommand(ac,fn,privdata,cmd,len); return status; } + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv) { + if (!ac->c.timeout) { + ac->c.timeout = calloc(1, sizeof(tv)); + } + + if (tv.tv_sec == ac->c.timeout->tv_sec && + tv.tv_usec == ac->c.timeout->tv_usec) { + return; + } + + *ac->c.timeout = tv; +} diff --git a/deps/hiredis/async.h b/deps/hiredis/async.h index 740555c24..4f6b3b783 100644 --- a/deps/hiredis/async.h +++ b/deps/hiredis/async.h @@ -57,6 +57,7 @@ typedef struct redisCallbackList { /* Connection callback prototypes */ typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); typedef void (redisConnectCallback)(const struct redisAsyncContext*, int status); +typedef void(redisTimerCallback)(void *timer, void *privdata); /* Context for an async connection to Redis */ typedef struct redisAsyncContext { @@ -81,6 +82,7 @@ typedef struct redisAsyncContext { void (*addWrite)(void *privdata); void (*delWrite)(void *privdata); void (*cleanup)(void *privdata); + void (*scheduleTimer)(void *privdata, struct timeval tv); } ev; /* Called when either the connection is terminated due to an error or per @@ -106,6 +108,7 @@ typedef struct redisAsyncContext { } redisAsyncContext; /* Functions that proxy to hiredis */ +redisAsyncContext *redisAsyncConnectWithOptions(const redisOptions *options); redisAsyncContext *redisAsyncConnect(const char *ip, int port); redisAsyncContext *redisAsyncConnectBind(const char *ip, int port, const char *source_addr); redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, @@ -113,12 +116,17 @@ redisAsyncContext *redisAsyncConnectBindWithReuse(const char *ip, int port, redisAsyncContext *redisAsyncConnectUnix(const char *path); int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); + +void redisAsyncSetTimeout(redisAsyncContext *ac, struct timeval tv); void redisAsyncDisconnect(redisAsyncContext *ac); void redisAsyncFree(redisAsyncContext *ac); /* Handle read/write events */ void redisAsyncHandleRead(redisAsyncContext *ac); void redisAsyncHandleWrite(redisAsyncContext *ac); +void redisAsyncHandleTimeout(redisAsyncContext *ac); +void redisAsyncRead(redisAsyncContext *ac); +void redisAsyncWrite(redisAsyncContext *ac); /* Command functions for an async context. Write the command to the * output buffer and register the provided callback. */ diff --git a/deps/hiredis/async_private.h b/deps/hiredis/async_private.h new file mode 100644 index 000000000..d0133ae18 --- /dev/null +++ b/deps/hiredis/async_private.h @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_ASYNC_PRIVATE_H +#define __HIREDIS_ASYNC_PRIVATE_H + +#define _EL_ADD_READ(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addRead) (ctx)->ev.addRead((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_READ(ctx) do { \ + if ((ctx)->ev.delRead) (ctx)->ev.delRead((ctx)->ev.data); \ + } while(0) +#define _EL_ADD_WRITE(ctx) \ + do { \ + refreshTimeout(ctx); \ + if ((ctx)->ev.addWrite) (ctx)->ev.addWrite((ctx)->ev.data); \ + } while (0) +#define _EL_DEL_WRITE(ctx) do { \ + if ((ctx)->ev.delWrite) (ctx)->ev.delWrite((ctx)->ev.data); \ + } while(0) +#define _EL_CLEANUP(ctx) do { \ + if ((ctx)->ev.cleanup) (ctx)->ev.cleanup((ctx)->ev.data); \ + ctx->ev.cleanup = NULL; \ + } while(0); + +static inline void refreshTimeout(redisAsyncContext *ctx) { + if (ctx->c.timeout && ctx->ev.scheduleTimer && + (ctx->c.timeout->tv_sec || ctx->c.timeout->tv_usec)) { + ctx->ev.scheduleTimer(ctx->ev.data, *ctx->c.timeout); + // } else { + // printf("Not scheduling timer.. (tmo=%p)\n", ctx->c.timeout); + // if (ctx->c.timeout){ + // printf("tv_sec: %u. tv_usec: %u\n", ctx->c.timeout->tv_sec, + // ctx->c.timeout->tv_usec); + // } + } +} + +void __redisAsyncDisconnect(redisAsyncContext *ac); +void redisProcessCallbacks(redisAsyncContext *ac); + +#endif /* __HIREDIS_ASYNC_PRIVATE_H */ diff --git a/deps/hiredis/examples/CMakeLists.txt b/deps/hiredis/examples/CMakeLists.txt new file mode 100644 index 000000000..dd3a313ac --- /dev/null +++ b/deps/hiredis/examples/CMakeLists.txt @@ -0,0 +1,46 @@ +INCLUDE(FindPkgConfig) +# Check for GLib + +PKG_CHECK_MODULES(GLIB2 glib-2.0) +if (GLIB2_FOUND) + INCLUDE_DIRECTORIES(${GLIB2_INCLUDE_DIRS}) + LINK_DIRECTORIES(${GLIB2_LIBRARY_DIRS}) + ADD_EXECUTABLE(example-glib example-glib.c) + TARGET_LINK_LIBRARIES(example-glib hiredis ${GLIB2_LIBRARIES}) +ENDIF(GLIB2_FOUND) + +FIND_PATH(LIBEV ev.h + HINTS /usr/local /usr/opt/local + ENV LIBEV_INCLUDE_DIR) + +if (LIBEV) + # Just compile and link with libev + ADD_EXECUTABLE(example-libev example-libev.c) + TARGET_LINK_LIBRARIES(example-libev hiredis ev) +ENDIF() + +FIND_PATH(LIBEVENT event.h) +if (LIBEVENT) + ADD_EXECUTABLE(example-libevent example-libevent) + TARGET_LINK_LIBRARIES(example-libevent hiredis event) +ENDIF() + +FIND_PATH(LIBUV uv.h) +IF (LIBUV) + ADD_EXECUTABLE(example-libuv example-libuv.c) + TARGET_LINK_LIBRARIES(example-libuv hiredis uv) +ENDIF() + +IF (APPLE) + FIND_LIBRARY(CF CoreFoundation) + ADD_EXECUTABLE(example-macosx example-macosx.c) + TARGET_LINK_LIBRARIES(example-macosx hiredis ${CF}) +ENDIF() + +IF (ENABLE_SSL) + ADD_EXECUTABLE(example-ssl example-ssl.c) + TARGET_LINK_LIBRARIES(example-ssl hiredis hiredis_ssl) +ENDIF() + +ADD_EXECUTABLE(example example.c) +TARGET_LINK_LIBRARIES(example hiredis) diff --git a/deps/hiredis/examples/example-libevent-ssl.c b/deps/hiredis/examples/example-libevent-ssl.c new file mode 100644 index 000000000..1021113b9 --- /dev/null +++ b/deps/hiredis/examples/example-libevent-ssl.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + +void getCallback(redisAsyncContext *c, void *r, void *privdata) { + redisReply *reply = r; + if (reply == NULL) return; + printf("argv[%s]: %s\n", (char*)privdata, reply->str); + + /* Disconnect after receiving the reply to GET */ + redisAsyncDisconnect(c); +} + +void connectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Connected...\n"); +} + +void disconnectCallback(const redisAsyncContext *c, int status) { + if (status != REDIS_OK) { + printf("Error: %s\n", c->errstr); + return; + } + printf("Disconnected...\n"); +} + +int main (int argc, char **argv) { + signal(SIGPIPE, SIG_IGN); + struct event_base *base = event_base_new(); + if (argc < 5) { + fprintf(stderr, + "Usage: %s [ca]\n", argv[0]); + exit(1); + } + + const char *value = argv[1]; + size_t nvalue = strlen(value); + + const char *hostname = argv[2]; + int port = atoi(argv[3]); + + const char *cert = argv[4]; + const char *certKey = argv[5]; + const char *caCert = argc > 5 ? argv[6] : NULL; + + redisAsyncContext *c = redisAsyncConnect(hostname, port); + if (c->err) { + /* Let *c leak for now... */ + printf("Error: %s\n", c->errstr); + return 1; + } + if (redisSecureConnection(&c->c, caCert, cert, certKey, "sni") != REDIS_OK) { + printf("SSL Error!\n"); + exit(1); + } + + redisLibeventAttach(c,base); + redisAsyncSetConnectCallback(c,connectCallback); + redisAsyncSetDisconnectCallback(c,disconnectCallback); + redisAsyncCommand(c, NULL, NULL, "SET key %b", value, nvalue); + redisAsyncCommand(c, getCallback, (char*)"end-1", "GET key"); + event_base_dispatch(base); + return 0; +} diff --git a/deps/hiredis/examples/example-libevent.c b/deps/hiredis/examples/example-libevent.c index d333c22b7..1fe71ae4e 100644 --- a/deps/hiredis/examples/example-libevent.c +++ b/deps/hiredis/examples/example-libevent.c @@ -9,7 +9,12 @@ void getCallback(redisAsyncContext *c, void *r, void *privdata) { redisReply *reply = r; - if (reply == NULL) return; + if (reply == NULL) { + if (c->errstr) { + printf("errstr: %s\n", c->errstr); + } + return; + } printf("argv[%s]: %s\n", (char*)privdata, reply->str); /* Disconnect after receiving the reply to GET */ @@ -35,8 +40,14 @@ void disconnectCallback(const redisAsyncContext *c, int status) { int main (int argc, char **argv) { signal(SIGPIPE, SIG_IGN); struct event_base *base = event_base_new(); + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, "127.0.0.1", 6379); + struct timeval tv = {0}; + tv.tv_sec = 1; + options.timeout = &tv; - redisAsyncContext *c = redisAsyncConnect("127.0.0.1", 6379); + + redisAsyncContext *c = redisAsyncConnectWithOptions(&options); if (c->err) { /* Let *c leak for now... */ printf("Error: %s\n", c->errstr); diff --git a/deps/hiredis/examples/example-ssl.c b/deps/hiredis/examples/example-ssl.c new file mode 100644 index 000000000..81f4648c6 --- /dev/null +++ b/deps/hiredis/examples/example-ssl.c @@ -0,0 +1,97 @@ +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) { + unsigned int j; + redisContext *c; + redisReply *reply; + if (argc < 4) { + printf("Usage: %s [ca]\n", argv[0]); + exit(1); + } + const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + int port = atoi(argv[2]); + const char *cert = argv[3]; + const char *key = argv[4]; + const char *ca = argc > 4 ? argv[5] : NULL; + + struct timeval tv = { 1, 500000 }; // 1.5 seconds + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, hostname, port); + options.timeout = &tv; + c = redisConnectWithOptions(&options); + + if (c == NULL || c->err) { + if (c) { + printf("Connection error: %s\n", c->errstr); + redisFree(c); + } else { + printf("Connection error: can't allocate redis context\n"); + } + exit(1); + } + + if (redisSecureConnection(c, ca, cert, key, "sni") != REDIS_OK) { + printf("Couldn't initialize SSL!\n"); + printf("Error: %s\n", c->errstr); + redisFree(c); + exit(1); + } + + /* PING server */ + reply = redisCommand(c,"PING"); + printf("PING: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key */ + reply = redisCommand(c,"SET %s %s", "foo", "hello world"); + printf("SET: %s\n", reply->str); + freeReplyObject(reply); + + /* Set a key using binary safe API */ + reply = redisCommand(c,"SET %b %b", "bar", (size_t) 3, "hello", (size_t) 5); + printf("SET (binary API): %s\n", reply->str); + freeReplyObject(reply); + + /* Try a GET and two INCR */ + reply = redisCommand(c,"GET foo"); + printf("GET foo: %s\n", reply->str); + freeReplyObject(reply); + + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + /* again ... */ + reply = redisCommand(c,"INCR counter"); + printf("INCR counter: %lld\n", reply->integer); + freeReplyObject(reply); + + /* Create a list of numbers, from 0 to 9 */ + reply = redisCommand(c,"DEL mylist"); + freeReplyObject(reply); + for (j = 0; j < 10; j++) { + char buf[64]; + + snprintf(buf,64,"%u",j); + reply = redisCommand(c,"LPUSH mylist element-%s", buf); + freeReplyObject(reply); + } + + /* Let's check what we have inside the list */ + reply = redisCommand(c,"LRANGE mylist 0 -1"); + if (reply->type == REDIS_REPLY_ARRAY) { + for (j = 0; j < reply->elements; j++) { + printf("%u) %s\n", j, reply->element[j]->str); + } + } + freeReplyObject(reply); + + /* Disconnects and frees the context */ + redisFree(c); + + return 0; +} diff --git a/deps/hiredis/examples/example.c b/deps/hiredis/examples/example.c index 4d494c55a..0e93fc8b3 100644 --- a/deps/hiredis/examples/example.c +++ b/deps/hiredis/examples/example.c @@ -5,14 +5,27 @@ #include int main(int argc, char **argv) { - unsigned int j; + unsigned int j, isunix = 0; redisContext *c; redisReply *reply; const char *hostname = (argc > 1) ? argv[1] : "127.0.0.1"; + + if (argc > 2) { + if (*argv[2] == 'u' || *argv[2] == 'U') { + isunix = 1; + /* in this case, host is the path to the unix socket */ + printf("Will connect to unix socket @%s\n", hostname); + } + } + int port = (argc > 2) ? atoi(argv[2]) : 6379; struct timeval timeout = { 1, 500000 }; // 1.5 seconds - c = redisConnectWithTimeout(hostname, port, timeout); + if (isunix) { + c = redisConnectUnixWithTimeout(hostname, timeout); + } else { + c = redisConnectWithTimeout(hostname, port, timeout); + } if (c == NULL || c->err) { if (c) { printf("Connection error: %s\n", c->errstr); diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 0947d1ed7..c4611318f 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -34,7 +34,6 @@ #include "fmacros.h" #include #include -#include #include #include #include @@ -42,10 +41,20 @@ #include "hiredis.h" #include "net.h" #include "sds.h" +#include "async.h" +#include "win32.h" + +static redisContextFuncs redisContextDefaultFuncs = { + .free_privdata = NULL, + .async_read = redisAsyncRead, + .async_write = redisAsyncWrite, + .read = redisNetRead, + .write = redisNetWrite +}; static redisReply *createReplyObject(int type); static void *createStringObject(const redisReadTask *task, char *str, size_t len); -static void *createArrayObject(const redisReadTask *task, int elements); +static void *createArrayObject(const redisReadTask *task, size_t elements); static void *createIntegerObject(const redisReadTask *task, long long value); static void *createDoubleObject(const redisReadTask *task, double value, char *str, size_t len); static void *createNilObject(const redisReadTask *task); @@ -138,7 +147,7 @@ static void *createStringObject(const redisReadTask *task, char *str, size_t len return r; } -static void *createArrayObject(const redisReadTask *task, int elements) { +static void *createArrayObject(const redisReadTask *task, size_t elements) { redisReply *r, *parent; r = createReplyObject(task->type); @@ -649,29 +658,30 @@ redisReader *redisReaderCreate(void) { return redisReaderCreateWithFunctions(&defaultFunctions); } -static redisContext *redisContextInit(void) { +static redisContext *redisContextInit(const redisOptions *options) { redisContext *c; - c = calloc(1,sizeof(redisContext)); + c = calloc(1, sizeof(*c)); if (c == NULL) return NULL; + c->funcs = &redisContextDefaultFuncs; c->obuf = sdsempty(); c->reader = redisReaderCreate(); + c->fd = REDIS_INVALID_FD; if (c->obuf == NULL || c->reader == NULL) { redisFree(c); return NULL; } - + (void)options; /* options are used in other functions */ return c; } void redisFree(redisContext *c) { if (c == NULL) return; - if (c->fd > 0) - close(c->fd); + redisNetClose(c); sdsfree(c->obuf); redisReaderFree(c->reader); @@ -680,12 +690,16 @@ void redisFree(redisContext *c) { free(c->unix_sock.path); free(c->timeout); free(c->saddr); + if (c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + } + memset(c, 0xff, sizeof(*c)); free(c); } -int redisFreeKeepFd(redisContext *c) { - int fd = c->fd; - c->fd = -1; +redisFD redisFreeKeepFd(redisContext *c) { + redisFD fd = c->fd; + c->fd = REDIS_INVALID_FD; redisFree(c); return fd; } @@ -694,10 +708,13 @@ int redisReconnect(redisContext *c) { c->err = 0; memset(c->errstr, '\0', strlen(c->errstr)); - if (c->fd > 0) { - close(c->fd); + if (c->privdata && c->funcs->free_privdata) { + c->funcs->free_privdata(c->privdata); + c->privdata = NULL; } + redisNetClose(c); + sdsfree(c->obuf); redisReaderFree(c->reader); @@ -718,112 +735,107 @@ int redisReconnect(redisContext *c) { return REDIS_ERR; } +redisContext *redisConnectWithOptions(const redisOptions *options) { + redisContext *c = redisContextInit(options); + if (c == NULL) { + return NULL; + } + if (!(options->options & REDIS_OPT_NONBLOCK)) { + c->flags |= REDIS_BLOCK; + } + if (options->options & REDIS_OPT_REUSEADDR) { + c->flags |= REDIS_REUSEADDR; + } + if (options->options & REDIS_OPT_NOAUTOFREE) { + c->flags |= REDIS_NO_AUTO_FREE; + } + + if (options->type == REDIS_CONN_TCP) { + redisContextConnectBindTcp(c, options->endpoint.tcp.ip, + options->endpoint.tcp.port, options->timeout, + options->endpoint.tcp.source_addr); + } else if (options->type == REDIS_CONN_UNIX) { + redisContextConnectUnix(c, options->endpoint.unix_socket, + options->timeout); + } else if (options->type == REDIS_CONN_USERFD) { + c->fd = options->endpoint.fd; + c->flags |= REDIS_CONNECTED; + } else { + // Unknown type - FIXME - FREE + return NULL; + } + if (options->timeout != NULL && (c->flags & REDIS_BLOCK) && c->fd != REDIS_INVALID_FD) { + redisContextSetTimeout(c, *options->timeout); + } + return c; +} + /* Connect to a Redis instance. On error the field error in the returned * context will be set to the return value of the error function. * When no set of reply functions is given, the default set will be used. */ redisContext *redisConnect(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + return redisConnectWithOptions(&options); } redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectNonBlock(const char *ip, int port) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectTcp(c,ip,port,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlock(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, const char *source_addr) { - redisContext *c = redisContextInit(); - if (c == NULL) - return NULL; - c->flags &= ~REDIS_BLOCK; - c->flags |= REDIS_REUSEADDR; - redisContextConnectBindTcp(c,ip,port,NULL,source_addr); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_TCP(&options, ip, port); + options.endpoint.tcp.source_addr = source_addr; + options.options |= REDIS_OPT_NONBLOCK|REDIS_OPT_REUSEADDR; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnix(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags |= REDIS_BLOCK; - redisContextConnectUnix(c,path,&tv); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.timeout = &tv; + return redisConnectWithOptions(&options); } redisContext *redisConnectUnixNonBlock(const char *path) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->flags &= ~REDIS_BLOCK; - redisContextConnectUnix(c,path,NULL); - return c; + redisOptions options = {0}; + REDIS_OPTIONS_SET_UNIX(&options, path); + options.options |= REDIS_OPT_NONBLOCK; + return redisConnectWithOptions(&options); } -redisContext *redisConnectFd(int fd) { - redisContext *c; - - c = redisContextInit(); - if (c == NULL) - return NULL; - - c->fd = fd; - c->flags |= REDIS_BLOCK | REDIS_CONNECTED; - return c; +redisContext *redisConnectFd(redisFD fd) { + redisOptions options = {0}; + options.type = REDIS_CONN_USERFD; + options.endpoint.fd = fd; + return redisConnectWithOptions(&options); } /* Set read/write timeout on a blocking socket. */ @@ -853,22 +865,15 @@ int redisBufferRead(redisContext *c) { if (c->err) return REDIS_ERR; - nread = read(c->fd,buf,sizeof(buf)); - if (nread == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ + nread = c->funcs->read(c, buf, sizeof(buf)); + if (nread > 0) { + if (redisReaderFeed(c->reader, buf, nread) != REDIS_OK) { + __redisSetError(c, c->reader->err, c->reader->errstr); + return REDIS_ERR; } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; } - } else if (nread == 0) { - __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection"); + } else if (nread < 0) { return REDIS_ERR; - } else { - if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) { - __redisSetError(c,c->reader->err,c->reader->errstr); - return REDIS_ERR; - } } return REDIS_OK; } @@ -883,21 +888,15 @@ int redisBufferRead(redisContext *c) { * c->errstr to hold the appropriate error string. */ int redisBufferWrite(redisContext *c, int *done) { - int nwritten; /* Return early when the context has seen an error. */ if (c->err) return REDIS_ERR; if (sdslen(c->obuf) > 0) { - nwritten = write(c->fd,c->obuf,sdslen(c->obuf)); - if (nwritten == -1) { - if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { - /* Try again later */ - } else { - __redisSetError(c,REDIS_ERR_IO,NULL); - return REDIS_ERR; - } + int nwritten = c->funcs->write(c); + if (nwritten < 0) { + return REDIS_ERR; } else if (nwritten > 0) { if (nwritten == (signed)sdslen(c->obuf)) { sdsfree(c->obuf); diff --git a/deps/hiredis/hiredis.h b/deps/hiredis/hiredis.h index 47d7982e9..f02678a47 100644 --- a/deps/hiredis/hiredis.h +++ b/deps/hiredis/hiredis.h @@ -35,7 +35,11 @@ #define __HIREDIS_H #include "read.h" #include /* for va_list */ +#ifndef _MSC_VER #include /* for struct timeval */ +#else +struct timeval; /* forward declaration */ +#endif #include /* uintXX_t, etc */ #include "sds.h" /* for sds */ @@ -74,6 +78,12 @@ /* Flag that is set when we should set SO_REUSEADDR before calling bind() */ #define REDIS_REUSEADDR 0x80 +/** + * Flag that indicates the user does not want the context to + * be automatically freed upon error + */ +#define REDIS_NO_AUTO_FREE 0x200 + #define REDIS_KEEPALIVE_INTERVAL 15 /* seconds */ /* number of times we retry to connect in the case of EADDRNOTAVAIL and @@ -111,14 +121,93 @@ void redisFreeSdsCommand(sds cmd); enum redisConnectionType { REDIS_CONN_TCP, - REDIS_CONN_UNIX + REDIS_CONN_UNIX, + REDIS_CONN_USERFD }; +struct redisSsl; + +#define REDIS_OPT_NONBLOCK 0x01 +#define REDIS_OPT_REUSEADDR 0x02 + +/** + * Don't automatically free the async object on a connection failure, + * or other implicit conditions. Only free on an explicit call to disconnect() or free() + */ +#define REDIS_OPT_NOAUTOFREE 0x04 + +/* In Unix systems a file descriptor is a regular signed int, with -1 + * representing an invalid descriptor. In Windows it is a SOCKET + * (32- or 64-bit unsigned integer depending on the architecture), where + * all bits set (~0) is INVALID_SOCKET. */ +#ifndef _WIN32 +typedef int redisFD; +#define REDIS_INVALID_FD -1 +#else +#ifdef _WIN64 +typedef unsigned long long redisFD; /* SOCKET = 64-bit UINT_PTR */ +#else +typedef unsigned long redisFD; /* SOCKET = 32-bit UINT_PTR */ +#endif +#define REDIS_INVALID_FD ((redisFD)(~0)) /* INVALID_SOCKET */ +#endif + +typedef struct { + /* + * the type of connection to use. This also indicates which + * `endpoint` member field to use + */ + int type; + /* bit field of REDIS_OPT_xxx */ + int options; + /* timeout value. if NULL, no timeout is used */ + const struct timeval *timeout; + union { + /** use this field for tcp/ip connections */ + struct { + const char *source_addr; + const char *ip; + int port; + } tcp; + /** use this field for unix domain sockets */ + const char *unix_socket; + /** + * use this field to have hiredis operate an already-open + * file descriptor */ + redisFD fd; + } endpoint; +} redisOptions; + +/** + * Helper macros to initialize options to their specified fields. + */ +#define REDIS_OPTIONS_SET_TCP(opts, ip_, port_) \ + (opts)->type = REDIS_CONN_TCP; \ + (opts)->endpoint.tcp.ip = ip_; \ + (opts)->endpoint.tcp.port = port_; + +#define REDIS_OPTIONS_SET_UNIX(opts, path) \ + (opts)->type = REDIS_CONN_UNIX; \ + (opts)->endpoint.unix_socket = path; + +struct redisAsyncContext; +struct redisContext; + +typedef struct redisContextFuncs { + void (*free_privdata)(void *); + void (*async_read)(struct redisAsyncContext *); + void (*async_write)(struct redisAsyncContext *); + int (*read)(struct redisContext *, char *, size_t); + int (*write)(struct redisContext *); +} redisContextFuncs; + /* Context for a connection to Redis */ typedef struct redisContext { + const redisContextFuncs *funcs; /* Function table */ + int err; /* Error flags, 0 when there is no error */ char errstr[128]; /* String representation of error when applicable */ - int fd; + redisFD fd; int flags; char *obuf; /* Write buffer */ redisReader *reader; /* Protocol reader */ @@ -139,8 +228,12 @@ typedef struct redisContext { /* For non-blocking connect */ struct sockadr *saddr; size_t addrlen; + + /* Additional private data for hiredis addons such as SSL */ + void *privdata; } redisContext; +redisContext *redisConnectWithOptions(const redisOptions *options); redisContext *redisConnect(const char *ip, int port); redisContext *redisConnectWithTimeout(const char *ip, int port, const struct timeval tv); redisContext *redisConnectNonBlock(const char *ip, int port); @@ -151,7 +244,7 @@ redisContext *redisConnectBindNonBlockWithReuse(const char *ip, int port, redisContext *redisConnectUnix(const char *path); redisContext *redisConnectUnixWithTimeout(const char *path, const struct timeval tv); redisContext *redisConnectUnixNonBlock(const char *path); -redisContext *redisConnectFd(int fd); +redisContext *redisConnectFd(redisFD fd); /** * Reconnect the given context using the saved information. @@ -167,7 +260,7 @@ int redisReconnect(redisContext *c); int redisSetTimeout(redisContext *c, const struct timeval tv); int redisEnableKeepAlive(redisContext *c); void redisFree(redisContext *c); -int redisFreeKeepFd(redisContext *c); +redisFD redisFreeKeepFd(redisContext *c); int redisBufferRead(redisContext *c); int redisBufferWrite(redisContext *c, int *done); diff --git a/deps/hiredis/hiredis.pc.in b/deps/hiredis/hiredis.pc.in new file mode 100644 index 000000000..140b040f1 --- /dev/null +++ b/deps/hiredis/hiredis.pc.in @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis +Description: Minimalistic C client library for Redis. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -lhiredis +Cflags: -I${pkgincludedir} -D_FILE_OFFSET_BITS=64 diff --git a/deps/hiredis/hiredis_ssl.h b/deps/hiredis/hiredis_ssl.h new file mode 100644 index 000000000..f844f9548 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.h @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __HIREDIS_SSL_H +#define __HIREDIS_SSL_H + +/* This is the underlying struct for SSL in ssl.h, which is not included to + * keep build dependencies short here. + */ +struct ssl_st; + +/** + * Secure the connection using SSL. This should be done before any command is + * executed on the connection. + */ +int redisSecureConnection(redisContext *c, const char *capath, const char *certpath, + const char *keypath, const char *servername); + +/** + * Initiate SSL/TLS negotiation on a provided context. + */ + +int redisInitiateSSL(redisContext *c, struct ssl_st *ssl); + +#endif /* __HIREDIS_SSL_H */ diff --git a/deps/hiredis/hiredis_ssl.pc.in b/deps/hiredis/hiredis_ssl.pc.in new file mode 100644 index 000000000..588a978a5 --- /dev/null +++ b/deps/hiredis/hiredis_ssl.pc.in @@ -0,0 +1,12 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/lib +includedir=${prefix}/include +pkgincludedir=${includedir}/hiredis + +Name: hiredis_ssl +Description: SSL Support for hiredis. +Version: @PROJECT_VERSION@ +Requires: hiredis +Libs: -L${libdir} -lhiredis_ssl +Libs.private: -lssl -lcrypto diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index a4b3abc6d..e5f40b0a4 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -34,36 +34,64 @@ #include "fmacros.h" #include -#include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include -#include #include #include #include "net.h" #include "sds.h" +#include "sockcompat.h" +#include "win32.h" /* Defined in hiredis.c */ void __redisSetError(redisContext *c, int type, const char *str); -static void redisContextCloseFd(redisContext *c) { - if (c && c->fd >= 0) { +void redisNetClose(redisContext *c) { + if (c && c->fd != REDIS_INVALID_FD) { close(c->fd); - c->fd = -1; + c->fd = REDIS_INVALID_FD; } } +int redisNetRead(redisContext *c, char *buf, size_t bufcap) { + int nread = recv(c->fd, buf, bufcap, 0); + if (nread == -1) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + return 0; + } else if(errno == ETIMEDOUT && (c->flags & REDIS_BLOCK)) { + /* especially in windows */ + __redisSetError(c, REDIS_ERR_TIMEOUT, "recv timeout"); + return -1; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + return nread; + } +} + +int redisNetWrite(redisContext *c) { + int nwritten = send(c->fd, c->obuf, sdslen(c->obuf), 0); + if (nwritten < 0) { + if ((errno == EWOULDBLOCK && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) { + /* Try again later */ + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return nwritten; +} + static void __redisSetErrorFromErrno(redisContext *c, int type, const char *prefix) { int errorno = errno; /* snprintf() may change errno */ char buf[128] = { 0 }; @@ -79,15 +107,15 @@ static int redisSetReuseAddr(redisContext *c) { int on = 1; if (setsockopt(c->fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; } static int redisCreateSocket(redisContext *c, int type) { - int s; - if ((s = socket(type, SOCK_STREAM, 0)) == -1) { + redisFD s; + if ((s = socket(type, SOCK_STREAM, 0)) == REDIS_INVALID_FD) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); return REDIS_ERR; } @@ -101,6 +129,7 @@ static int redisCreateSocket(redisContext *c, int type) { } static int redisSetBlocking(redisContext *c, int blocking) { +#ifndef _WIN32 int flags; /* Set the socket nonblocking. @@ -108,7 +137,7 @@ static int redisSetBlocking(redisContext *c, int blocking) { * interrupted by a signal. */ if ((flags = fcntl(c->fd, F_GETFL)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_GETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -119,15 +148,23 @@ static int redisSetBlocking(redisContext *c, int blocking) { if (fcntl(c->fd, F_SETFL, flags) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"fcntl(F_SETFL)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } +#else + u_long mode = blocking ? 0 : 1; + if (ioctl(c->fd, FIONBIO, &mode) == -1) { + __redisSetErrorFromErrno(c, REDIS_ERR_IO, "ioctl(FIONBIO)"); + redisNetClose(c); + return REDIS_ERR; + } +#endif /* _WIN32 */ return REDIS_OK; } int redisKeepAlive(redisContext *c, int interval) { int val = 1; - int fd = c->fd; + redisFD fd = c->fd; if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1){ __redisSetError(c,REDIS_ERR_OTHER,strerror(errno)); @@ -170,7 +207,7 @@ static int redisSetTcpNoDelay(redisContext *c) { int yes = 1; if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(TCP_NODELAY)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } return REDIS_OK; @@ -212,12 +249,12 @@ static int redisContextWaitReady(redisContext *c, long msec) { if ((res = poll(wfd, 1, msec)) == -1) { __redisSetErrorFromErrno(c, REDIS_ERR_IO, "poll(2)"); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } else if (res == 0) { errno = ETIMEDOUT; __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -230,7 +267,7 @@ static int redisContextWaitReady(redisContext *c, long msec) { } __redisSetErrorFromErrno(c,REDIS_ERR_IO,NULL); - redisContextCloseFd(c); + redisNetClose(c); return REDIS_ERR; } @@ -277,11 +314,18 @@ int redisCheckSocketError(redisContext *c) { } int redisContextSetTimeout(redisContext *c, const struct timeval tv) { - if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,&tv,sizeof(tv)) == -1) { + const void *to_ptr = &tv; + size_t to_sz = sizeof(tv); +#ifdef _WIN32 + DWORD timeout_msec = tv.tv_sec * 1000 + tv.tv_usec / 1000; + to_ptr = &timeout_msec; + to_sz = sizeof(timeout_msec); +#endif + if (setsockopt(c->fd,SOL_SOCKET,SO_RCVTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_RCVTIMEO)"); return REDIS_ERR; } - if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,&tv,sizeof(tv)) == -1) { + if (setsockopt(c->fd,SOL_SOCKET,SO_SNDTIMEO,to_ptr,to_sz) == -1) { __redisSetErrorFromErrno(c,REDIS_ERR_IO,"setsockopt(SO_SNDTIMEO)"); return REDIS_ERR; } @@ -291,7 +335,8 @@ int redisContextSetTimeout(redisContext *c, const struct timeval tv) { static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout, const char *source_addr) { - int s, rv, n; + redisFD s; + int rv, n; char _port[6]; /* strlen("65535"); */ struct addrinfo hints, *servinfo, *bservinfo, *p, *b; int blocking = (c->flags & REDIS_BLOCK); @@ -360,7 +405,7 @@ static int _redisContextConnectTcp(redisContext *c, const char *addr, int port, } for (p = servinfo; p != NULL; p = p->ai_next) { addrretry: - if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == REDIS_INVALID_FD) continue; c->fd = s; @@ -401,16 +446,14 @@ addrretry: } /* For repeat connection */ - if (c->saddr) { - free(c->saddr); - } + free(c->saddr); c->saddr = malloc(p->ai_addrlen); memcpy(c->saddr, p->ai_addr, p->ai_addrlen); c->addrlen = p->ai_addrlen; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { if (errno == EHOSTUNREACH) { - redisContextCloseFd(c); + redisNetClose(c); continue; } else if (errno == EINPROGRESS) { if (blocking) { @@ -424,7 +467,7 @@ addrretry: if (++reuses >= REDIS_CONNECT_RETRIES) { goto error; } else { - redisContextCloseFd(c); + redisNetClose(c); goto addrretry; } } else { @@ -471,8 +514,9 @@ int redisContextConnectBindTcp(redisContext *c, const char *addr, int port, } int redisContextConnectUnix(redisContext *c, const char *path, const struct timeval *timeout) { +#ifndef _WIN32 int blocking = (c->flags & REDIS_BLOCK); - struct sockaddr_un sa; + struct sockaddr_un *sa; long timeout_msec = -1; if (redisCreateSocket(c,AF_UNIX) < 0) @@ -499,9 +543,11 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time if (redisContextTimeoutMsec(c,&timeout_msec) != REDIS_OK) return REDIS_ERR; - sa.sun_family = AF_UNIX; - strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); - if (connect(c->fd, (struct sockaddr*)&sa, sizeof(sa)) == -1) { + sa = (struct sockaddr_un*)(c->saddr = malloc(sizeof(struct sockaddr_un))); + c->addrlen = sizeof(struct sockaddr_un); + sa->sun_family = AF_UNIX; + strncpy(sa->sun_path, path, sizeof(sa->sun_path) - 1); + if (connect(c->fd, (struct sockaddr*)sa, sizeof(*sa)) == -1) { if (errno == EINPROGRESS && !blocking) { /* This is ok. */ } else { @@ -516,4 +562,10 @@ int redisContextConnectUnix(redisContext *c, const char *path, const struct time c->flags |= REDIS_CONNECTED; return REDIS_OK; +#else + /* We currently do not support Unix sockets for Windows. */ + /* TODO(m): https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/ */ + errno = EPROTONOSUPPORT; + return REDIS_ERR; +#endif /* _WIN32 */ } diff --git a/deps/hiredis/net.h b/deps/hiredis/net.h index a11594e68..a4393c06b 100644 --- a/deps/hiredis/net.h +++ b/deps/hiredis/net.h @@ -37,6 +37,10 @@ #include "hiredis.h" +void redisNetClose(redisContext *c); +int redisNetRead(redisContext *c, char *buf, size_t bufcap); +int redisNetWrite(redisContext *c); + int redisCheckSocketError(redisContext *c); int redisContextSetTimeout(redisContext *c, const struct timeval tv); int redisContextConnectTcp(redisContext *c, const char *addr, int port, const struct timeval *timeout); diff --git a/deps/hiredis/read.c b/deps/hiredis/read.c index cc0f3cc72..dfe196352 100644 --- a/deps/hiredis/read.c +++ b/deps/hiredis/read.c @@ -31,10 +31,10 @@ #include "fmacros.h" #include -#include #include #ifndef _MSC_VER #include +#include #endif #include #include @@ -44,6 +44,7 @@ #include "read.h" #include "sds.h" +#include "win32.h" static void __redisReaderSetError(redisReader *r, int type, const char *str) { size_t len; @@ -294,9 +295,9 @@ static int processLineItem(redisReader *r) { buf[len] = '\0'; if (strcasecmp(buf,",inf") == 0) { - d = 1.0/0.0; /* Positive infinite. */ + d = INFINITY; /* Positive infinite. */ } else if (strcasecmp(buf,",-inf") == 0) { - d = -1.0/0.0; /* Nevative infinite. */ + d = -INFINITY; /* Nevative infinite. */ } else { d = strtod((char*)buf,&eptr); if (buf[0] == '\0' || eptr[0] != '\0' || isnan(d)) { @@ -430,7 +431,7 @@ static int processAggregateItem(redisReader *r) { root = (r->ridx == 0); - if (elements < -1 || elements > INT_MAX) { + if (elements < -1 || (LLONG_MAX > SIZE_MAX && elements > SIZE_MAX)) { __redisReaderSetError(r,REDIS_ERR_PROTOCOL, "Multi-bulk length out of range"); return REDIS_ERR; @@ -657,8 +658,11 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Emit a reply when there is one. */ if (r->ridx == -1) { - if (reply != NULL) + if (reply != NULL) { *reply = r->reply; + } else if (r->reply != NULL && r->fn && r->fn->freeObject) { + r->fn->freeObject(r->reply); + } r->reply = NULL; } return REDIS_OK; diff --git a/deps/hiredis/read.h b/deps/hiredis/read.h index f3d075843..af02aaf68 100644 --- a/deps/hiredis/read.h +++ b/deps/hiredis/read.h @@ -45,6 +45,7 @@ #define REDIS_ERR_EOF 3 /* End of file */ #define REDIS_ERR_PROTOCOL 4 /* Protocol error */ #define REDIS_ERR_OOM 5 /* Out of memory */ +#define REDIS_ERR_TIMEOUT 6 /* Timed out */ #define REDIS_ERR_OTHER 2 /* Everything else... */ #define REDIS_REPLY_STRING 1 @@ -79,7 +80,7 @@ typedef struct redisReadTask { typedef struct redisReplyObjectFunctions { void *(*createString)(const redisReadTask*, char*, size_t); - void *(*createArray)(const redisReadTask*, int); + void *(*createArray)(const redisReadTask*, size_t); void *(*createInteger)(const redisReadTask*, long long); void *(*createDouble)(const redisReadTask*, double, char*, size_t); void *(*createNil)(const redisReadTask*); diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 44777b10c..6cf75841c 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1035,7 +1035,7 @@ sds *sdssplitargs(const char *line, int *argc) { s_free(vector); return NULL; } - + vector = new_vector; vector[*argc] = current; (*argc)++; diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 13be75a9f..3f9a96457 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -34,6 +34,9 @@ #define __SDS_H #define SDS_MAX_PREALLOC (1024*1024) +#ifdef _MSC_VER +#define __attribute__(x) +#endif #include #include @@ -132,20 +135,20 @@ static inline void sdssetlen(sds s, size_t newlen) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); + *fp = (unsigned char)(SDS_TYPE_5 | (newlen << SDS_TYPE_BITS)); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len = newlen; + SDS_HDR(8,s)->len = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len = newlen; + SDS_HDR(16,s)->len = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len = newlen; + SDS_HDR(32,s)->len = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len = newlen; + SDS_HDR(64,s)->len = (uint64_t)newlen; break; } } @@ -156,21 +159,21 @@ static inline void sdsinclen(sds s, size_t inc) { case SDS_TYPE_5: { unsigned char *fp = ((unsigned char*)s)-1; - unsigned char newlen = SDS_TYPE_5_LEN(flags)+inc; + unsigned char newlen = SDS_TYPE_5_LEN(flags)+(unsigned char)inc; *fp = SDS_TYPE_5 | (newlen << SDS_TYPE_BITS); } break; case SDS_TYPE_8: - SDS_HDR(8,s)->len += inc; + SDS_HDR(8,s)->len += (uint8_t)inc; break; case SDS_TYPE_16: - SDS_HDR(16,s)->len += inc; + SDS_HDR(16,s)->len += (uint16_t)inc; break; case SDS_TYPE_32: - SDS_HDR(32,s)->len += inc; + SDS_HDR(32,s)->len += (uint32_t)inc; break; case SDS_TYPE_64: - SDS_HDR(64,s)->len += inc; + SDS_HDR(64,s)->len += (uint64_t)inc; break; } } @@ -200,16 +203,16 @@ static inline void sdssetalloc(sds s, size_t newlen) { /* Nothing to do, this type has no total allocation info. */ break; case SDS_TYPE_8: - SDS_HDR(8,s)->alloc = newlen; + SDS_HDR(8,s)->alloc = (uint8_t)newlen; break; case SDS_TYPE_16: - SDS_HDR(16,s)->alloc = newlen; + SDS_HDR(16,s)->alloc = (uint16_t)newlen; break; case SDS_TYPE_32: - SDS_HDR(32,s)->alloc = newlen; + SDS_HDR(32,s)->alloc = (uint32_t)newlen; break; case SDS_TYPE_64: - SDS_HDR(64,s)->alloc = newlen; + SDS_HDR(64,s)->alloc = (uint64_t)newlen; break; } } diff --git a/deps/hiredis/sockcompat.c b/deps/hiredis/sockcompat.c new file mode 100644 index 000000000..4cc2f414f --- /dev/null +++ b/deps/hiredis/sockcompat.c @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDIS_SOCKCOMPAT_IMPLEMENTATION +#include "sockcompat.h" + +#ifdef _WIN32 +static int _wsaErrorToErrno(int err) { + switch (err) { + case WSAEWOULDBLOCK: + return EWOULDBLOCK; + case WSAEINPROGRESS: + return EINPROGRESS; + case WSAEALREADY: + return EALREADY; + case WSAENOTSOCK: + return ENOTSOCK; + case WSAEDESTADDRREQ: + return EDESTADDRREQ; + case WSAEMSGSIZE: + return EMSGSIZE; + case WSAEPROTOTYPE: + return EPROTOTYPE; + case WSAENOPROTOOPT: + return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: + return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: + return EOPNOTSUPP; + case WSAEAFNOSUPPORT: + return EAFNOSUPPORT; + case WSAEADDRINUSE: + return EADDRINUSE; + case WSAEADDRNOTAVAIL: + return EADDRNOTAVAIL; + case WSAENETDOWN: + return ENETDOWN; + case WSAENETUNREACH: + return ENETUNREACH; + case WSAENETRESET: + return ENETRESET; + case WSAECONNABORTED: + return ECONNABORTED; + case WSAECONNRESET: + return ECONNRESET; + case WSAENOBUFS: + return ENOBUFS; + case WSAEISCONN: + return EISCONN; + case WSAENOTCONN: + return ENOTCONN; + case WSAETIMEDOUT: + return ETIMEDOUT; + case WSAECONNREFUSED: + return ECONNREFUSED; + case WSAELOOP: + return ELOOP; + case WSAENAMETOOLONG: + return ENAMETOOLONG; + case WSAEHOSTUNREACH: + return EHOSTUNREACH; + case WSAENOTEMPTY: + return ENOTEMPTY; + default: + /* We just return a generic I/O error if we could not find a relevant error. */ + return EIO; + } +} + +static void _updateErrno(int success) { + errno = success ? 0 : _wsaErrorToErrno(WSAGetLastError()); +} + +static int _initWinsock() { + static int s_initialized = 0; + if (!s_initialized) { + static WSADATA wsadata; + int err = WSAStartup(MAKEWORD(2,2), &wsadata); + if (err != 0) { + errno = _wsaErrorToErrno(err); + return 0; + } + s_initialized = 1; + } + return 1; +} + +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return EAI_FAIL; + } + + switch (getaddrinfo(node, service, hints, res)) { + case 0: return 0; + case WSATRY_AGAIN: return EAI_AGAIN; + case WSAEINVAL: return EAI_BADFLAGS; + case WSAEAFNOSUPPORT: return EAI_FAMILY; + case WSA_NOT_ENOUGH_MEMORY: return EAI_MEMORY; + case WSAHOST_NOT_FOUND: return EAI_NONAME; + case WSATYPE_NOT_FOUND: return EAI_SERVICE; + case WSAESOCKTNOSUPPORT: return EAI_SOCKTYPE; + default: return EAI_FAIL; /* Including WSANO_RECOVERY */ + } +} + +const char *win32_gai_strerror(int errcode) { + switch (errcode) { + case 0: errcode = 0; break; + case EAI_AGAIN: errcode = WSATRY_AGAIN; break; + case EAI_BADFLAGS: errcode = WSAEINVAL; break; + case EAI_FAMILY: errcode = WSAEAFNOSUPPORT; break; + case EAI_MEMORY: errcode = WSA_NOT_ENOUGH_MEMORY; break; + case EAI_NONAME: errcode = WSAHOST_NOT_FOUND; break; + case EAI_SERVICE: errcode = WSATYPE_NOT_FOUND; break; + case EAI_SOCKTYPE: errcode = WSAESOCKTNOSUPPORT; break; + default: errcode = WSANO_RECOVERY; break; /* Including EAI_FAIL */ + } + return gai_strerror(errcode); +} + +void win32_freeaddrinfo(struct addrinfo *res) { + freeaddrinfo(res); +} + +SOCKET win32_socket(int domain, int type, int protocol) { + SOCKET s; + + /* Note: This function is likely to be called before other functions, so run init here. */ + if (!_initWinsock()) { + return INVALID_SOCKET; + } + + _updateErrno((s = socket(domain, type, protocol)) != INVALID_SOCKET); + return s; +} + +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp) { + int ret = ioctlsocket(fd, (long)request, argp); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = bind(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen) { + int ret = connect(sockfd, addr, addrlen); + _updateErrno(ret != SOCKET_ERROR); + + /* For Winsock connect(), the WSAEWOULDBLOCK error means the same thing as + * EINPROGRESS for POSIX connect(), so we do that translation to keep POSIX + * logic consistent. */ + if (errno == EWOULDBLOCK) { + errno = EINPROGRESS; + } + + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + if (*optlen >= sizeof (struct timeval)) { + struct timeval *tv = optval; + DWORD timeout = 0; + socklen_t dwlen = 0; + ret = getsockopt(sockfd, level, optname, (char *)&timeout, &dwlen); + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout * 1000) % 1000000; + } else { + ret = WSAEFAULT; + } + *optlen = sizeof (struct timeval); + } else { + ret = getsockopt(sockfd, level, optname, (char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen) { + int ret = 0; + if ((level == SOL_SOCKET) && ((optname == SO_RCVTIMEO) || (optname == SO_SNDTIMEO))) { + struct timeval *tv = optval; + DWORD timeout = tv->tv_sec * 1000 + tv->tv_usec / 1000; + ret = setsockopt(sockfd, level, optname, (const char*)&timeout, sizeof(DWORD)); + } else { + ret = setsockopt(sockfd, level, optname, (const char*)optval, optlen); + } + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_close(SOCKET fd) { + int ret = closesocket(fd); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags) { + int ret = recv(sockfd, (char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags) { + int ret = send(sockfd, (const char*)buf, (int)len, flags); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} + +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout) { + int ret = WSAPoll(fds, nfds, timeout); + _updateErrno(ret != SOCKET_ERROR); + return ret != SOCKET_ERROR ? ret : -1; +} +#endif /* _WIN32 */ diff --git a/deps/hiredis/sockcompat.h b/deps/hiredis/sockcompat.h new file mode 100644 index 000000000..56006c163 --- /dev/null +++ b/deps/hiredis/sockcompat.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2019, Marcus Geelnard + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SOCKCOMPAT_H +#define __SOCKCOMPAT_H + +#ifndef _WIN32 +/* For POSIX systems we use the standard BSD socket API. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#else +/* For Windows we use winsock. */ +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x0600 /* To get WSAPoll etc. */ +#include +#include +#include + +#ifdef _MSC_VER +typedef signed long ssize_t; +#endif + +/* Emulate the parts of the BSD socket API that we need (override the winsock signatures). */ +int win32_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res); +const char *win32_gai_strerror(int errcode); +void win32_freeaddrinfo(struct addrinfo *res); +SOCKET win32_socket(int domain, int type, int protocol); +int win32_ioctl(SOCKET fd, unsigned long request, unsigned long *argp); +int win32_bind(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_connect(SOCKET sockfd, const struct sockaddr *addr, socklen_t addrlen); +int win32_getsockopt(SOCKET sockfd, int level, int optname, void *optval, socklen_t *optlen); +int win32_setsockopt(SOCKET sockfd, int level, int optname, const void *optval, socklen_t optlen); +int win32_close(SOCKET fd); +ssize_t win32_recv(SOCKET sockfd, void *buf, size_t len, int flags); +ssize_t win32_send(SOCKET sockfd, const void *buf, size_t len, int flags); +typedef ULONG nfds_t; +int win32_poll(struct pollfd *fds, nfds_t nfds, int timeout); + +#ifndef REDIS_SOCKCOMPAT_IMPLEMENTATION +#define getaddrinfo(node, service, hints, res) win32_getaddrinfo(node, service, hints, res) +#undef gai_strerror +#define gai_strerror(errcode) win32_gai_strerror(errcode) +#define freeaddrinfo(res) win32_freeaddrinfo(res) +#define socket(domain, type, protocol) win32_socket(domain, type, protocol) +#define ioctl(fd, request, argp) win32_ioctl(fd, request, argp) +#define bind(sockfd, addr, addrlen) win32_bind(sockfd, addr, addrlen) +#define connect(sockfd, addr, addrlen) win32_connect(sockfd, addr, addrlen) +#define getsockopt(sockfd, level, optname, optval, optlen) win32_getsockopt(sockfd, level, optname, optval, optlen) +#define setsockopt(sockfd, level, optname, optval, optlen) win32_setsockopt(sockfd, level, optname, optval, optlen) +#define close(fd) win32_close(fd) +#define recv(sockfd, buf, len, flags) win32_recv(sockfd, buf, len, flags) +#define send(sockfd, buf, len, flags) win32_send(sockfd, buf, len, flags) +#define poll(fds, nfds, timeout) win32_poll(fds, nfds, timeout) +#endif /* REDIS_SOCKCOMPAT_IMPLEMENTATION */ +#endif /* _WIN32 */ + +#endif /* __SOCKCOMPAT_H */ diff --git a/deps/hiredis/ssl.c b/deps/hiredis/ssl.c new file mode 100644 index 000000000..78ab9e43e --- /dev/null +++ b/deps/hiredis/ssl.c @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2009-2011, Salvatore Sanfilippo + * Copyright (c) 2010-2011, Pieter Noordhuis + * Copyright (c) 2019, Redis Labs + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "hiredis.h" +#include "async.h" + +#include +#include +#include +#include + +#include +#include + +#include "async_private.h" + +void __redisSetError(redisContext *c, int type, const char *str); + +/* The SSL context is attached to SSL/TLS connections as a privdata. */ +typedef struct redisSSLContext { + /** + * OpenSSL SSL_CTX; It is optional and will not be set when using + * user-supplied SSL. + */ + SSL_CTX *ssl_ctx; + + /** + * OpenSSL SSL object. + */ + SSL *ssl; + + /** + * SSL_write() requires to be called again with the same arguments it was + * previously called with in the event of an SSL_read/SSL_write situation + */ + size_t lastLen; + + /** Whether the SSL layer requires read (possibly before a write) */ + int wantRead; + + /** + * Whether a write was requested prior to a read. If set, the write() + * should resume whenever a read takes place, if possible + */ + int pendingWrite; +} redisSSLContext; + +/* Forward declaration */ +redisContextFuncs redisContextSSLFuncs; + +#ifdef HIREDIS_SSL_TRACE +/** + * Callback used for debugging + */ +static void sslLogCallback(const SSL *ssl, int where, int ret) { + const char *retstr = ""; + int should_log = 1; + /* Ignore low-level SSL stuff */ + + if (where & SSL_CB_ALERT) { + should_log = 1; + } + if (where == SSL_CB_HANDSHAKE_START || where == SSL_CB_HANDSHAKE_DONE) { + should_log = 1; + } + if ((where & SSL_CB_EXIT) && ret == 0) { + should_log = 1; + } + + if (!should_log) { + return; + } + + retstr = SSL_alert_type_string(ret); + printf("ST(0x%x). %s. R(0x%x)%s\n", where, SSL_state_string_long(ssl), ret, retstr); + + if (where == SSL_CB_HANDSHAKE_DONE) { + printf("Using SSL version %s. Cipher=%s\n", SSL_get_version(ssl), SSL_get_cipher_name(ssl)); + } +} +#endif + +/** + * OpenSSL global initialization and locking handling callbacks. + * Note that this is only required for OpenSSL < 1.1.0. + */ + +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HIREDIS_USE_CRYPTO_LOCKS +#endif + +#ifdef HIREDIS_USE_CRYPTO_LOCKS +typedef pthread_mutex_t sslLockType; +static void sslLockInit(sslLockType *l) { + pthread_mutex_init(l, NULL); +} +static void sslLockAcquire(sslLockType *l) { + pthread_mutex_lock(l); +} +static void sslLockRelease(sslLockType *l) { + pthread_mutex_unlock(l); +} +static pthread_mutex_t *ossl_locks; + +static void opensslDoLock(int mode, int lkid, const char *f, int line) { + sslLockType *l = ossl_locks + lkid; + + if (mode & CRYPTO_LOCK) { + sslLockAcquire(l); + } else { + sslLockRelease(l); + } + + (void)f; + (void)line; +} + +static void initOpensslLocks(void) { + unsigned ii, nlocks; + if (CRYPTO_get_locking_callback() != NULL) { + /* Someone already set the callback before us. Don't destroy it! */ + return; + } + nlocks = CRYPTO_num_locks(); + ossl_locks = malloc(sizeof(*ossl_locks) * nlocks); + for (ii = 0; ii < nlocks; ii++) { + sslLockInit(ossl_locks + ii); + } + CRYPTO_set_locking_callback(opensslDoLock); +} +#endif /* HIREDIS_USE_CRYPTO_LOCKS */ + +/** + * SSL Connection initialization. + */ + +static int redisSSLConnect(redisContext *c, SSL_CTX *ssl_ctx, SSL *ssl) { + if (c->privdata) { + __redisSetError(c, REDIS_ERR_OTHER, "redisContext was already associated"); + return REDIS_ERR; + } + c->privdata = calloc(1, sizeof(redisSSLContext)); + + c->funcs = &redisContextSSLFuncs; + redisSSLContext *rssl = c->privdata; + + rssl->ssl_ctx = ssl_ctx; + rssl->ssl = ssl; + + SSL_set_mode(rssl->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_set_fd(rssl->ssl, c->fd); + SSL_set_connect_state(rssl->ssl); + + ERR_clear_error(); + int rv = SSL_connect(rssl->ssl); + if (rv == 1) { + return REDIS_OK; + } + + rv = SSL_get_error(rssl->ssl, rv); + if (((c->flags & REDIS_BLOCK) == 0) && + (rv == SSL_ERROR_WANT_READ || rv == SSL_ERROR_WANT_WRITE)) { + return REDIS_OK; + } + + if (c->err == 0) { + char err[512]; + if (rv == SSL_ERROR_SYSCALL) + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s",strerror(errno)); + else { + unsigned long e = ERR_peek_last_error(); + snprintf(err,sizeof(err)-1,"SSL_connect failed: %s", + ERR_reason_error_string(e)); + } + __redisSetError(c, REDIS_ERR_IO, err); + } + return REDIS_ERR; +} + +int redisInitiateSSL(redisContext *c, SSL *ssl) { + return redisSSLConnect(c, NULL, ssl); +} + +int redisSecureConnection(redisContext *c, const char *capath, + const char *certpath, const char *keypath, const char *servername) { + + SSL_CTX *ssl_ctx = NULL; + SSL *ssl = NULL; + + /* Initialize global OpenSSL stuff */ + static int isInit = 0; + if (!isInit) { + isInit = 1; + SSL_library_init(); +#ifdef HIREDIS_USE_CRYPTO_LOCKS + initOpensslLocks(); +#endif + } + + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + __redisSetError(c, REDIS_ERR_OTHER, "Failed to create SSL_CTX"); + goto error; + } + +#ifdef HIREDIS_SSL_TRACE + SSL_CTX_set_info_callback(ssl_ctx, sslLogCallback); +#endif + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + if ((certpath != NULL && keypath == NULL) || (keypath != NULL && certpath == NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "certpath and keypath must be specified together"); + goto error; + } + + if (capath) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, capath, NULL)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid CA certificate"); + goto error; + } + } + if (certpath) { + if (!SSL_CTX_use_certificate_chain_file(ssl_ctx, certpath)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client certificate"); + goto error; + } + if (!SSL_CTX_use_PrivateKey_file(ssl_ctx, keypath, SSL_FILETYPE_PEM)) { + __redisSetError(c, REDIS_ERR_OTHER, "Invalid client key"); + goto error; + } + } + + ssl = SSL_new(ssl_ctx); + if (!ssl) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't create new SSL instance"); + goto error; + } + if (servername) { + if (!SSL_set_tlsext_host_name(ssl, servername)) { + __redisSetError(c, REDIS_ERR_OTHER, "Couldn't set server name indication"); + goto error; + } + } + + return redisSSLConnect(c, ssl_ctx, ssl); + +error: + if (ssl) SSL_free(ssl); + if (ssl_ctx) SSL_CTX_free(ssl_ctx); + return REDIS_ERR; +} + +static int maybeCheckWant(redisSSLContext *rssl, int rv) { + /** + * If the error is WANT_READ or WANT_WRITE, the appropriate flags are set + * and true is returned. False is returned otherwise + */ + if (rv == SSL_ERROR_WANT_READ) { + rssl->wantRead = 1; + return 1; + } else if (rv == SSL_ERROR_WANT_WRITE) { + rssl->pendingWrite = 1; + return 1; + } else { + return 0; + } +} + +/** + * Implementation of redisContextFuncs for SSL connections. + */ + +static void redisSSLFreeContext(void *privdata){ + redisSSLContext *rsc = privdata; + + if (!rsc) return; + if (rsc->ssl) { + SSL_free(rsc->ssl); + rsc->ssl = NULL; + } + if (rsc->ssl_ctx) { + SSL_CTX_free(rsc->ssl_ctx); + rsc->ssl_ctx = NULL; + } + free(rsc); +} + +static int redisSSLRead(redisContext *c, char *buf, size_t bufcap) { + redisSSLContext *rssl = c->privdata; + + int nread = SSL_read(rssl->ssl, buf, bufcap); + if (nread > 0) { + return nread; + } else if (nread == 0) { + __redisSetError(c, REDIS_ERR_EOF, "Server closed the connection"); + return -1; + } else { + int err = SSL_get_error(rssl->ssl, nread); + if (c->flags & REDIS_BLOCK) { + /** + * In blocking mode, we should never end up in a situation where + * we get an error without it being an actual error, except + * in the case of EINTR, which can be spuriously received from + * debuggers or whatever. + */ + if (errno == EINTR) { + return 0; + } else { + const char *msg = NULL; + if (errno == EAGAIN) { + msg = "Resource temporarily unavailable"; + } + __redisSetError(c, REDIS_ERR_IO, msg); + return -1; + } + } + + /** + * We can very well get an EWOULDBLOCK/EAGAIN, however + */ + if (maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } +} + +static int redisSSLWrite(redisContext *c) { + redisSSLContext *rssl = c->privdata; + + size_t len = rssl->lastLen ? rssl->lastLen : sdslen(c->obuf); + int rv = SSL_write(rssl->ssl, c->obuf, len); + + if (rv > 0) { + rssl->lastLen = 0; + } else if (rv < 0) { + rssl->lastLen = len; + + int err = SSL_get_error(rssl->ssl, rv); + if ((c->flags & REDIS_BLOCK) == 0 && maybeCheckWant(rssl, err)) { + return 0; + } else { + __redisSetError(c, REDIS_ERR_IO, NULL); + return -1; + } + } + return rv; +} + +static void redisSSLAsyncRead(redisAsyncContext *ac) { + int rv; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->wantRead = 0; + + if (rssl->pendingWrite) { + int done; + + /* This is probably just a write event */ + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } else if (!done) { + _EL_ADD_WRITE(ac); + } + } + + rv = redisBufferRead(c); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + } else { + _EL_ADD_READ(ac); + redisProcessCallbacks(ac); + } +} + +static void redisSSLAsyncWrite(redisAsyncContext *ac) { + int rv, done = 0; + redisSSLContext *rssl = ac->c.privdata; + redisContext *c = &ac->c; + + rssl->pendingWrite = 0; + rv = redisBufferWrite(c, &done); + if (rv == REDIS_ERR) { + __redisAsyncDisconnect(ac); + return; + } + + if (!done) { + if (rssl->wantRead) { + /* Need to read-before-write */ + rssl->pendingWrite = 1; + _EL_DEL_WRITE(ac); + } else { + /* No extra reads needed, just need to write more */ + _EL_ADD_WRITE(ac); + } + } else { + /* Already done! */ + _EL_DEL_WRITE(ac); + } + + /* Always reschedule a read */ + _EL_ADD_READ(ac); +} + +redisContextFuncs redisContextSSLFuncs = { + .free_privdata = redisSSLFreeContext, + .async_read = redisSSLAsyncRead, + .async_write = redisSSLAsyncWrite, + .read = redisSSLRead, + .write = redisSSLWrite +}; + diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 79cff4308..8668e1856 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -13,12 +13,16 @@ #include #include "hiredis.h" +#ifdef HIREDIS_TEST_SSL +#include "hiredis_ssl.h" +#endif #include "net.h" enum connection_type { CONN_TCP, CONN_UNIX, - CONN_FD + CONN_FD, + CONN_SSL }; struct config { @@ -33,6 +37,14 @@ struct config { struct { const char *path; } unix_sock; + + struct { + const char *host; + int port; + const char *ca_cert; + const char *cert; + const char *key; + } ssl; }; /* The following lines make up our testing "framework" :) */ @@ -93,11 +105,27 @@ static int disconnect(redisContext *c, int keep_fd) { return -1; } +static void do_ssl_handshake(redisContext *c, struct config config) { +#ifdef HIREDIS_TEST_SSL + redisSecureConnection(c, config.ssl.ca_cert, config.ssl.cert, config.ssl.key, NULL); + if (c->err) { + printf("SSL error: %s\n", c->errstr); + redisFree(c); + exit(1); + } +#else + (void) c; + (void) config; +#endif +} + static redisContext *do_connect(struct config config) { redisContext *c = NULL; if (config.type == CONN_TCP) { c = redisConnect(config.tcp.host, config.tcp.port); + } else if (config.type == CONN_SSL) { + c = redisConnect(config.ssl.host, config.ssl.port); } else if (config.type == CONN_UNIX) { c = redisConnectUnix(config.unix_sock.path); } else if (config.type == CONN_FD) { @@ -121,9 +149,21 @@ static redisContext *do_connect(struct config config) { exit(1); } + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } + return select_database(c); } +static void do_reconnect(redisContext *c, struct config config) { + redisReconnect(c); + + if (config.type == CONN_SSL) { + do_ssl_handshake(c, config); + } +} + static void test_format_commands(void) { char *cmd; int len; @@ -360,7 +400,8 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); - test("Set error when array > INT_MAX: "); +#if LLONG_MAX > SIZE_MAX + test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "*9223372036854775807\r\n+asdf\r\n",29); ret = redisReaderGetReply(reader,&reply); @@ -369,7 +410,6 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); -#if LLONG_MAX > SIZE_MAX test("Set error when bulk > SIZE_MAX: "); reader = redisReaderCreate(); redisReaderFeed(reader, "$9223372036854775807\r\nasdf\r\n",28); @@ -434,22 +474,23 @@ static void test_free_null(void) { test_cond(reply == NULL); } +#define HIREDIS_BAD_DOMAIN "idontexist-noreally.com" static void test_blocking_connection_errors(void) { redisContext *c; struct addrinfo hints = {.ai_family = AF_INET}; struct addrinfo *ai_tmp = NULL; - const char *bad_domain = "idontexist.com"; - int rv = getaddrinfo(bad_domain, "6379", &hints, &ai_tmp); + int rv = getaddrinfo(HIREDIS_BAD_DOMAIN, "6379", &hints, &ai_tmp); if (rv != 0) { // Address does *not* exist test("Returns error when host cannot be resolved: "); // First see if this domain name *actually* resolves to NXDOMAIN - c = redisConnect("dontexist.com", 6379); + c = redisConnect(HIREDIS_BAD_DOMAIN, 6379); test_cond( c->err == REDIS_ERR_OTHER && (strcmp(c->errstr, "Name or service not known") == 0 || - strcmp(c->errstr, "Can't resolve: sadkfjaskfjsa.com") == 0 || + strcmp(c->errstr, "Can't resolve: " HIREDIS_BAD_DOMAIN) == 0 || + strcmp(c->errstr, "Name does not resolve") == 0 || strcmp(c->errstr, "nodename nor servname provided, or not known") == 0 || strcmp(c->errstr, "No address associated with hostname") == 0 || @@ -574,7 +615,8 @@ static void test_blocking_connection_timeouts(struct config config) { c = do_connect(config); test("Does not return a reply when the command times out: "); - s = write(c->fd, cmd, strlen(cmd)); + redisAppendFormattedCommand(c, cmd, strlen(cmd)); + s = c->funcs->write(c); tv.tv_sec = 0; tv.tv_usec = 10000; redisSetTimeout(c, tv); @@ -583,7 +625,7 @@ static void test_blocking_connection_timeouts(struct config config) { freeReplyObject(reply); test("Reconnect properly reconnects after a timeout: "); - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -591,7 +633,7 @@ static void test_blocking_connection_timeouts(struct config config) { test("Reconnect properly uses owned parameters: "); config.tcp.host = "foo"; config.unix_sock.path = "foo"; - redisReconnect(c); + do_reconnect(c, config); reply = redisCommand(c, "PING"); test_cond(reply != NULL && reply->type == REDIS_REPLY_STATUS && strcmp(reply->str, "PONG") == 0); freeReplyObject(reply); @@ -894,6 +936,23 @@ int main(int argc, char **argv) { throughput = 0; } else if (argc >= 1 && !strcmp(argv[0],"--skip-inherit-fd")) { test_inherit_fd = 0; +#ifdef HIREDIS_TEST_SSL + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-port")) { + argv++; argc--; + cfg.ssl.port = atoi(argv[0]); + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-host")) { + argv++; argc--; + cfg.ssl.host = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-ca-cert")) { + argv++; argc--; + cfg.ssl.ca_cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-cert")) { + argv++; argc--; + cfg.ssl.cert = argv[0]; + } else if (argc >= 2 && !strcmp(argv[0],"--ssl-key")) { + argv++; argc--; + cfg.ssl.key = argv[0]; +#endif } else { fprintf(stderr, "Invalid argument: %s\n", argv[0]); exit(1); @@ -922,6 +981,20 @@ int main(int argc, char **argv) { test_blocking_io_errors(cfg); if (throughput) test_throughput(cfg); +#ifdef HIREDIS_TEST_SSL + if (cfg.ssl.port && cfg.ssl.host) { + printf("\nTesting against SSL connection (%s:%d):\n", cfg.ssl.host, cfg.ssl.port); + cfg.type = CONN_SSL; + + test_blocking_connection(cfg); + test_blocking_connection_timeouts(cfg); + test_blocking_io_errors(cfg); + test_invalid_timeout_errors(cfg); + test_append_formatted_commands(cfg); + if (throughput) test_throughput(cfg); + } +#endif + if (test_inherit_fd) { printf("\nTesting against inherited fd (%s):\n", cfg.unix_sock.path); cfg.type = CONN_FD; diff --git a/deps/hiredis/test.sh b/deps/hiredis/test.sh new file mode 100755 index 000000000..2cab9e6fb --- /dev/null +++ b/deps/hiredis/test.sh @@ -0,0 +1,70 @@ +#!/bin/sh -ue + +REDIS_SERVER=${REDIS_SERVER:-redis-server} +REDIS_PORT=${REDIS_PORT:-56379} +REDIS_SSL_PORT=${REDIS_SSL_PORT:-56443} +TEST_SSL=${TEST_SSL:-0} +SSL_TEST_ARGS= + +tmpdir=$(mktemp -d) +PID_FILE=${tmpdir}/hiredis-test-redis.pid +SOCK_FILE=${tmpdir}/hiredis-test-redis.sock + +if [ "$TEST_SSL" = "1" ]; then + SSL_CA_CERT=${tmpdir}/ca.crt + SSL_CA_KEY=${tmpdir}/ca.key + SSL_CERT=${tmpdir}/redis.crt + SSL_KEY=${tmpdir}/redis.key + + openssl genrsa -out ${tmpdir}/ca.key 4096 + openssl req \ + -x509 -new -nodes -sha256 \ + -key ${SSL_CA_KEY} \ + -days 3650 \ + -subj '/CN=Hiredis Test CA' \ + -out ${SSL_CA_CERT} + openssl genrsa -out ${SSL_KEY} 2048 + openssl req \ + -new -sha256 \ + -key ${SSL_KEY} \ + -subj '/CN=Hiredis Test Cert' | \ + openssl x509 \ + -req -sha256 \ + -CA ${SSL_CA_CERT} \ + -CAkey ${SSL_CA_KEY} \ + -CAserial ${tmpdir}/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out ${SSL_CERT} + + SSL_TEST_ARGS="--ssl-host 127.0.0.1 --ssl-port ${REDIS_SSL_PORT} --ssl-ca-cert ${SSL_CA_CERT} --ssl-cert ${SSL_CERT} --ssl-key ${SSL_KEY}" +fi + +cleanup() { + set +e + kill $(cat ${PID_FILE}) + rm -rf ${tmpdir} +} +trap cleanup INT TERM EXIT + +cat > ${tmpdir}/redis.conf <> ${tmpdir}/redis.conf < /* for struct timeval */ + #ifndef inline #define inline __inline #endif +#ifndef strcasecmp +#define strcasecmp stricmp +#endif + +#ifndef strncasecmp +#define strncasecmp strnicmp +#endif + #ifndef va_copy #define va_copy(d,s) ((d) = (s)) #endif @@ -37,6 +47,10 @@ __inline int c99_snprintf(char* str, size_t size, const char* format, ...) return count; } #endif +#endif /* _MSC_VER */ -#endif -#endif \ No newline at end of file +#ifdef _WIN32 +#define strerror_r(errno,buf,len) strerror_s(buf,len,errno) +#endif /* _WIN32 */ + +#endif /* _WIN32_HELPER_INCLUDE */ diff --git a/redis.conf b/redis.conf index 50ba823ac..2af422a93 100644 --- a/redis.conf +++ b/redis.conf @@ -129,6 +129,50 @@ timeout 0 # Redis default starting with Redis 3.2.1. tcp-keepalive 300 +################################# TLS/SSL ##################################### + +# By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration +# directive can be used to define TLS-listening ports. To enable TLS on the +# default port, use: +# +# port 0 +# tls-port 6379 + +# Configure a X.509 certificate and private key to use for authenticating the +# server to connected clients, masters or cluster peers. These files should be +# PEM formatted. +# +# tls-cert-file redis.crt tls-key-file redis.key + +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# +# tls-dh-params-file redis.dh + +# Configure a CA certificate(s) bundle to authenticate TLS/SSL clients and +# peers. +# +# tls-ca-cert-file ca.crt + +# If TLS/SSL clients are required to authenticate using a client side +# certificate, use this directive. +# +# Note: this applies to all incoming clients, including replicas. +# +# tls-auth-clients yes + +# If TLS/SSL should be used when connecting as a replica to a master, enable +# this configuration directive: +# +# tls-replication yes + +# If TLS/SSL should be used for the Redis Cluster bus, enable this configuration +# directive. +# +# NOTE: If TLS/SSL is enabled for Cluster Bus, mutual authentication is always +# enforced. +# +# tls-cluster yes + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. diff --git a/src/Makefile b/src/Makefile index b6cc69e2f..b43b743dc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -145,6 +145,12 @@ ifeq ($(MALLOC),jemalloc) FINAL_LIBS := ../deps/jemalloc/lib/libjemalloc.a $(FINAL_LIBS) endif +ifeq ($(BUILD_TLS),yes) + FINAL_CFLAGS+=-DUSE_OPENSSL $(OPENSSL_CFLAGS) + FINAL_LDFLAGS+=$(OPENSSL_LDFLAGS) + FINAL_LIBS += ../deps/hiredis/libhiredis_ssl.a -lssl -lcrypto +endif + REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) @@ -164,7 +170,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o connection.o tls.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark diff --git a/src/anet.c b/src/anet.c index 2088f4fb1..46ea7e145 100644 --- a/src/anet.c +++ b/src/anet.c @@ -279,8 +279,8 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONE 0 #define ANET_CONNECT_NONBLOCK 1 #define ANET_CONNECT_BE_BINDING 2 /* Best effort binding. */ -static int anetTcpGenericConnect(char *err, char *addr, int port, - char *source_addr, int flags) +static int anetTcpGenericConnect(char *err, const char *addr, int port, + const char *source_addr, int flags) { int s = ANET_ERR, rv; char portstr[6]; /* strlen("65535") + 1; */ @@ -359,31 +359,31 @@ end: } } -int anetTcpConnect(char *err, char *addr, int port) +int anetTcpConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONE); } -int anetTcpNonBlockConnect(char *err, char *addr, int port) +int anetTcpNonBlockConnect(char *err, const char *addr, int port) { return anetTcpGenericConnect(err,addr,port,NULL,ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK); } -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, - char *source_addr) +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, + const char *source_addr) { return anetTcpGenericConnect(err,addr,port,source_addr, ANET_CONNECT_NONBLOCK|ANET_CONNECT_BE_BINDING); } -int anetUnixGenericConnect(char *err, char *path, int flags) +int anetUnixGenericConnect(char *err, const char *path, int flags) { int s; struct sockaddr_un sa; @@ -411,12 +411,12 @@ int anetUnixGenericConnect(char *err, char *path, int flags) return s; } -int anetUnixConnect(char *err, char *path) +int anetUnixConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONE); } -int anetUnixNonBlockConnect(char *err, char *path) +int anetUnixNonBlockConnect(char *err, const char *path) { return anetUnixGenericConnect(err,path,ANET_CONNECT_NONBLOCK); } diff --git a/src/anet.h b/src/anet.h index dd735240d..23f19643c 100644 --- a/src/anet.h +++ b/src/anet.h @@ -49,12 +49,12 @@ #undef ip_len #endif -int anetTcpConnect(char *err, char *addr, int port); -int anetTcpNonBlockConnect(char *err, char *addr, int port); -int anetTcpNonBlockBindConnect(char *err, char *addr, int port, char *source_addr); -int anetTcpNonBlockBestEffortBindConnect(char *err, char *addr, int port, char *source_addr); -int anetUnixConnect(char *err, char *path); -int anetUnixNonBlockConnect(char *err, char *path); +int anetTcpConnect(char *err, const char *addr, int port); +int anetTcpNonBlockConnect(char *err, const char *addr, int port); +int anetTcpNonBlockBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetTcpNonBlockBestEffortBindConnect(char *err, const char *addr, int port, const char *source_addr); +int anetUnixConnect(char *err, const char *path); +int anetUnixNonBlockConnect(char *err, const char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); diff --git a/src/aof.c b/src/aof.c index 7237cdfbc..eed994bf2 100644 --- a/src/aof.c +++ b/src/aof.c @@ -653,7 +653,7 @@ struct client *createFakeClient(void) { struct client *c = zmalloc(sizeof(*c)); selectDb(c,0); - c->fd = -1; + c->conn = NULL; c->name = NULL; c->querybuf = sdsempty(); c->querybuf_peak = 0; diff --git a/src/cluster.c b/src/cluster.c index a2615fdc0..639ab1ea7 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -49,7 +49,7 @@ clusterNode *myself = NULL; clusterNode *createClusterNode(char *nodename, int flags); int clusterAddNode(clusterNode *node); void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); +void clusterReadHandler(connection *conn); void clusterSendPing(clusterLink *link, int type); void clusterSendFail(char *nodename); void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request); @@ -477,7 +477,8 @@ void clusterInit(void) { /* Port sanity check II * The other handshake port check is triggered too late to stop * us from trying to use a too-high cluster port number. */ - if (server.port > (65535-CLUSTER_PORT_INCR)) { + int port = server.tls_cluster ? server.tls_port : server.port; + if (port > (65535-CLUSTER_PORT_INCR)) { serverLog(LL_WARNING, "Redis port number too high. " "Cluster communication port is 10,000 port " "numbers higher than your Redis port. " @@ -485,8 +486,7 @@ void clusterInit(void) { "lower than 55535."); exit(1); } - - if (listenToPort(server.port+CLUSTER_PORT_INCR, + if (listenToPort(port+CLUSTER_PORT_INCR, server.cfd,&server.cfd_count) == C_ERR) { exit(1); @@ -508,8 +508,8 @@ void clusterInit(void) { /* Set myself->port / cport to my listening ports, we'll just need to * discover the IP address via MEET messages. */ - myself->port = server.port; - myself->cport = server.port+CLUSTER_PORT_INCR; + myself->port = port; + myself->cport = port+CLUSTER_PORT_INCR; if (server.cluster_announce_port) myself->port = server.cluster_announce_port; if (server.cluster_announce_bus_port) @@ -593,7 +593,7 @@ clusterLink *createClusterLink(clusterNode *node) { link->sndbuf = sdsempty(); link->rcvbuf = sdsempty(); link->node = node; - link->fd = -1; + link->conn = NULL; return link; } @@ -601,23 +601,45 @@ clusterLink *createClusterLink(clusterNode *node) { * This function will just make sure that the original node associated * with this link will have the 'link' field set to NULL. */ void freeClusterLink(clusterLink *link) { - if (link->fd != -1) { - aeDeleteFileEvent(server.el, link->fd, AE_READABLE|AE_WRITABLE); + if (link->conn) { + connClose(link->conn); + link->conn = NULL; } sdsfree(link->sndbuf); sdsfree(link->rcvbuf); if (link->node) link->node->link = NULL; - close(link->fd); zfree(link); } +static void clusterConnAcceptHandler(connection *conn) { + clusterLink *link; + + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", connGetLastError(conn)); + connClose(conn); + return; + } + + /* Create a link object we use to handle the connection. + * It gets passed to the readable handler when data is available. + * Initiallly the link->node pointer is set to NULL as we don't know + * which node is, but the right node is references once we know the + * node identity. */ + link = createClusterLink(NULL); + link->conn = conn; + connSetPrivateData(conn, link); + + /* Register read handler */ + connSetReadHandler(conn, clusterReadHandler); +} + #define MAX_CLUSTER_ACCEPTS_PER_CALL 1000 void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; int max = MAX_CLUSTER_ACCEPTS_PER_CALL; char cip[NET_IP_STR_LEN]; - clusterLink *link; UNUSED(el); UNUSED(mask); UNUSED(privdata); @@ -634,19 +656,24 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { "Error accepting cluster node: %s", server.neterr); return; } - anetNonBlock(NULL,cfd); - anetEnableTcpNoDelay(NULL,cfd); + + connection *conn = server.tls_cluster ? connCreateAcceptedTLS(cfd,1) : connCreateAcceptedSocket(cfd); + connNonBlock(conn); + connEnableTcpNoDelay(conn); /* Use non-blocking I/O for cluster messages. */ - serverLog(LL_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* Create a link object we use to handle the connection. - * It gets passed to the readable handler when data is available. - * Initiallly the link->node pointer is set to NULL as we don't know - * which node is, but the right node is references once we know the - * node identity. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(server.el,cfd,AE_READABLE,clusterReadHandler,link); + serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport); + + /* Accept the connection now. connAccept() may call our handler directly + * or schedule it for later depending on connection implementation. + */ + if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) { + serverLog(LL_VERBOSE, + "Error accepting cluster node connection: %s", + connGetLastError(conn)); + connClose(conn); + return; + } } } @@ -1447,7 +1474,7 @@ void nodeIp2String(char *buf, clusterLink *link, char *announced_ip) { memcpy(buf,announced_ip,NET_IP_STR_LEN); buf[NET_IP_STR_LEN-1] = '\0'; /* We are not sure the input is sane. */ } else { - anetPeerToString(link->fd, buf, NET_IP_STR_LEN, NULL); + connPeerToString(link->conn, buf, NET_IP_STR_LEN, NULL); } } @@ -1751,7 +1778,7 @@ int clusterProcessPacket(clusterLink *link) { { char ip[NET_IP_STR_LEN]; - if (anetSockName(link->fd,ip,sizeof(ip),NULL) != -1 && + if (connSockName(link->conn,ip,sizeof(ip),NULL) != -1 && strcmp(ip,myself->ip)) { memcpy(myself->ip,ip,NET_IP_STR_LEN); @@ -2118,35 +2145,76 @@ void handleLinkIOError(clusterLink *link) { /* Send data. This is handled using a trivial send buffer that gets * consumed by write(). We don't try to optimize this for speed too much * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - clusterLink *link = (clusterLink*) privdata; +void clusterWriteHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); ssize_t nwritten; - UNUSED(el); - UNUSED(mask); - nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf)); + nwritten = connWrite(conn, link->sndbuf, sdslen(link->sndbuf)); if (nwritten <= 0) { serverLog(LL_DEBUG,"I/O error writing to node link: %s", - (nwritten == -1) ? strerror(errno) : "short write"); + (nwritten == -1) ? connGetLastError(conn) : "short write"); handleLinkIOError(link); return; } sdsrange(link->sndbuf,nwritten,-1); if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); + connSetWriteHandler(link->conn, NULL); +} + +/* A connect handler that gets called when a connection to another node + * gets established. + */ +void clusterLinkConnectHandler(connection *conn) { + clusterLink *link = connGetPrivateData(conn); + clusterNode *node = link->node; + + /* Check if connection succeeded */ + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_VERBOSE, "Connection with Node %.40s at %s:%d failed: %s", + node->name, node->ip, node->cport, + connGetLastError(conn)); + freeClusterLink(link); + return; + } + + /* Register a read handler from now on */ + connSetReadHandler(conn, clusterReadHandler); + + /* Queue a PING in the new connection ASAP: this is crucial + * to avoid false positives in failure detection. + * + * If the node is flagged as MEET, we send a MEET message instead + * of a PING one, to force the receiver to add us in its node + * table. */ + mstime_t old_ping_sent = node->ping_sent; + clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? + CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); + if (old_ping_sent) { + /* If there was an active ping before the link was + * disconnected, we want to restore the ping time, otherwise + * replaced by the clusterSendPing() call. */ + node->ping_sent = old_ping_sent; + } + /* We can clear the flag after the first packet is sent. + * If we'll never receive a PONG, we'll never send new packets + * to this node. Instead after the PONG is received and we + * are no longer in meet/handshake status, we want to send + * normal PING packets. */ + node->flags &= ~CLUSTER_NODE_MEET; + + serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", + node->name, node->ip, node->cport); } /* Read data. Try to read the first field of the header first to check the * full length of the packet. When a whole packet is in memory this function * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { +void clusterReadHandler(connection *conn) { char buf[sizeof(clusterMsg)]; ssize_t nread; clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; + clusterLink *link = connGetPrivateData(conn); unsigned int readlen, rcvbuflen; - UNUSED(el); - UNUSED(mask); while(1) { /* Read as long as there is data to read. */ rcvbuflen = sdslen(link->rcvbuf); @@ -2174,13 +2242,13 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { if (readlen > sizeof(buf)) readlen = sizeof(buf); } - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* No more data ready. */ + nread = connRead(conn,buf,readlen); + if (nread == -1 && (connGetState(conn) == CONN_STATE_CONNECTED)) return; /* No more data ready. */ if (nread <= 0) { /* I/O error... */ serverLog(LL_DEBUG,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); + (nread == 0) ? "connection closed" : connGetLastError(conn)); handleLinkIOError(link); return; } else { @@ -2209,8 +2277,7 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { * from event handlers that will do stuff with the same link later. */ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE|AE_BARRIER, - clusterWriteHandler,link); + connSetWriteHandler(link->conn, clusterWriteHandler); /* TODO: Handle AE_BARRIER in conns */ link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); @@ -2276,11 +2343,12 @@ void clusterBuildMessageHdr(clusterMsg *hdr, int type) { } /* Handle cluster-announce-port as well. */ + int port = server.tls_cluster ? server.tls_port : server.port; int announced_port = server.cluster_announce_port ? - server.cluster_announce_port : server.port; + server.cluster_announce_port : port; int announced_cport = server.cluster_announce_bus_port ? server.cluster_announce_bus_port : - (server.port + CLUSTER_PORT_INCR); + (port + CLUSTER_PORT_INCR); memcpy(hdr->myslots,master->slots,sizeof(hdr->myslots)); memset(hdr->slaveof,0,CLUSTER_NAMELEN); @@ -3383,13 +3451,11 @@ void clusterCron(void) { } if (node->link == NULL) { - int fd; - mstime_t old_ping_sent; - clusterLink *link; - - fd = anetTcpNonBlockBindConnect(server.neterr, node->ip, - node->cport, NET_FIRST_BIND_ADDR); - if (fd == -1) { + clusterLink *link = createClusterLink(node); + link->conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + connSetPrivateData(link->conn, link); + if (connConnect(link->conn, node->ip, node->cport, NET_FIRST_BIND_ADDR, + clusterLinkConnectHandler) == -1) { /* We got a synchronous error from connect before * clusterSendPing() had a chance to be called. * If node->ping_sent is zero, failure detection can't work, @@ -3399,37 +3465,11 @@ void clusterCron(void) { serverLog(LL_DEBUG, "Unable to connect to " "Cluster Node [%s]:%d -> %s", node->ip, node->cport, server.neterr); + + freeClusterLink(link); continue; } - link = createClusterLink(node); - link->fd = fd; node->link = link; - aeCreateFileEvent(server.el,link->fd,AE_READABLE, - clusterReadHandler,link); - /* Queue a PING in the new connection ASAP: this is crucial - * to avoid false positives in failure detection. - * - * If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - old_ping_sent = node->ping_sent; - clusterSendPing(link, node->flags & CLUSTER_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - if (old_ping_sent) { - /* If there was an active ping before the link was - * disconnected, we want to restore the ping time, otherwise - * replaced by the clusterSendPing() call. */ - node->ping_sent = old_ping_sent; - } - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~CLUSTER_NODE_MEET; - - serverLog(LL_DEBUG,"Connecting with Node %.40s at %s:%d", - node->name, node->ip, node->cport); } } dictReleaseIterator(di); @@ -4940,7 +4980,7 @@ void restoreCommand(client *c) { #define MIGRATE_SOCKET_CACHE_TTL 10 /* close cached sockets after 10 sec. */ typedef struct migrateCachedSocket { - int fd; + connection *conn; long last_dbid; time_t last_use_time; } migrateCachedSocket; @@ -4957,7 +4997,7 @@ typedef struct migrateCachedSocket { * should be called so that the connection will be created from scratch * the next time. */ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long timeout) { - int fd; + connection *conn; sds name = sdsempty(); migrateCachedSocket *cs; @@ -4977,34 +5017,27 @@ migrateCachedSocket* migrateGetSocket(client *c, robj *host, robj *port, long ti /* Too many items, drop one at random. */ dictEntry *de = dictGetRandomKey(server.migrate_cached_sockets); cs = dictGetVal(de); - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } /* Create the socket */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); - if (fd == -1) { - sdsfree(name); - addReplyErrorFormat(c,"Can't connect to target node: %s", - server.neterr); - return NULL; - } - anetEnableTcpNoDelay(server.neterr,fd); - - /* Check if it connects within the specified timeout. */ - if ((aeWait(fd,AE_WRITABLE,timeout) & AE_WRITABLE) == 0) { - sdsfree(name); + conn = server.tls_cluster ? connCreateTLS() : connCreateSocket(); + if (connBlockingConnect(conn, c->argv[1]->ptr, atoi(c->argv[2]->ptr), timeout) + != C_OK) { addReplySds(c, sdsnew("-IOERR error or timeout connecting to the client\r\n")); - close(fd); + connClose(conn); + sdsfree(name); return NULL; } + connEnableTcpNoDelay(conn); /* Add to the cache and return it to the caller. */ cs = zmalloc(sizeof(*cs)); - cs->fd = fd; + cs->conn = conn; + cs->last_dbid = -1; cs->last_use_time = server.unixtime; dictAdd(server.migrate_cached_sockets,name,cs); @@ -5025,7 +5058,7 @@ void migrateCloseSocket(robj *host, robj *port) { return; } - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,name); sdsfree(name); @@ -5039,7 +5072,7 @@ void migrateCloseTimedoutSockets(void) { migrateCachedSocket *cs = dictGetVal(de); if ((server.unixtime - cs->last_use_time) > MIGRATE_SOCKET_CACHE_TTL) { - close(cs->fd); + connClose(cs->conn); zfree(cs); dictDelete(server.migrate_cached_sockets,dictGetKey(de)); } @@ -5221,7 +5254,7 @@ try_again: while ((towrite = sdslen(buf)-pos) > 0) { towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(cs->fd,buf+pos,towrite,timeout); + nwritten = connSyncWrite(cs->conn,buf+pos,towrite,timeout); if (nwritten != (signed)towrite) { write_error = 1; goto socket_err; @@ -5235,11 +5268,11 @@ try_again: char buf2[1024]; /* Restore reply. */ /* Read the AUTH reply if needed. */ - if (password && syncReadLine(cs->fd, buf0, sizeof(buf0), timeout) <= 0) + if (password && connSyncReadLine(cs->conn, buf0, sizeof(buf0), timeout) <= 0) goto socket_err; /* Read the SELECT reply if needed. */ - if (select && syncReadLine(cs->fd, buf1, sizeof(buf1), timeout) <= 0) + if (select && connSyncReadLine(cs->conn, buf1, sizeof(buf1), timeout) <= 0) goto socket_err; /* Read the RESTORE replies. */ @@ -5254,7 +5287,7 @@ try_again: if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); for (j = 0; j < num_keys; j++) { - if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { + if (connSyncReadLine(cs->conn, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; } diff --git a/src/cluster.h b/src/cluster.h index 571b9c543..ffbb29f0d 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -40,7 +40,7 @@ struct clusterNode; /* clusterLink encapsulates everything needed to talk with a remote node. */ typedef struct clusterLink { mstime_t ctime; /* Link creation time */ - int fd; /* TCP socket file descriptor */ + connection *conn; /* Connection to remote node */ sds sndbuf; /* Packet send buffer */ sds rcvbuf; /* Packet reception buffer */ struct clusterNode *node; /* Node related to this link if any, or NULL */ diff --git a/src/config.c b/src/config.c index a72df2e78..456fb0226 100644 --- a/src/config.c +++ b/src/config.c @@ -286,6 +286,15 @@ void loadServerConfigFromString(char *config) { if (server.port < 0 || server.port > 65535) { err = "Invalid port"; goto loaderr; } + } else if (!strcasecmp(argv[0],"tls-port") && argc == 2) { +#ifdef USE_OPENSSL + server.tls_port = atoi(argv[1]); + if (server.port < 0 || server.port > 65535) { + err = "Invalid port"; goto loaderr; + } +#else + err = "TLS not supported"; goto loaderr; +#endif } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { server.tcp_backlog = atoi(argv[1]); if (server.tcp_backlog < 0) { @@ -791,6 +800,24 @@ void loadServerConfigFromString(char *config) { err = sentinelHandleConfiguration(argv+1,argc-1); if (err) goto loaderr; } + } else if (!strcasecmp(argv[0],"tls-cert-file") && argc == 2) { + zfree(server.tls_cert_file); + server.tls_cert_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-key-file") && argc == 2) { + zfree(server.tls_key_file); + server.tls_key_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-dh-params-file") && argc == 2) { + zfree(server.tls_dh_params_file); + server.tls_dh_params_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-ca-cert-file") && argc == 2) { + zfree(server.tls_ca_cert_file); + server.tls_ca_cert_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-cluster") && argc == 2) { + server.tls_cluster = yesnotoi(argv[1]); + } else if (!strcasecmp(argv[0],"tls-replication") && argc == 2) { + server.tls_replication = yesnotoi(argv[1]); + } else if (!strcasecmp(argv[0],"tls-auth-clients") && argc == 2) { + server.tls_auth_clients = yesnotoi(argv[1]); } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -1234,6 +1261,45 @@ void configSetCommand(client *c) { } config_set_enum_field( "repl-diskless-load",server.repl_diskless_load,repl_diskless_load_enum) { + /* TLS fields. */ + } config_set_special_field("tls-cert-file") { + if (tlsConfigure((char *) o->ptr, server.tls_key_file, + server.tls_dh_params_file, server.tls_ca_cert_file) == C_ERR) { + addReplyError(c, + "Unable to configure tls-cert-file. Check server logs."); + return; + } + zfree(server.tls_cert_file); + server.tls_cert_file = zstrdup(o->ptr); + } config_set_special_field("tls-key-file") { + if (tlsConfigure(server.tls_cert_file, (char *) o->ptr, + server.tls_dh_params_file, server.tls_ca_cert_file) == C_ERR) { + addReplyError(c, + "Unable to configure tls-key-file. Check server logs."); + return; + } + zfree(server.tls_key_file); + server.tls_key_file = zstrdup(o->ptr); + } config_set_special_field("tls-dh-params-file") { + if (tlsConfigure(server.tls_cert_file, server.tls_key_file, + (char *) o->ptr, server.tls_ca_cert_file) == C_ERR) { + addReplyError(c, + "Unable to configure tls-dh-params-file. Check server logs."); + return; + } + zfree(server.tls_dh_params_file); + server.tls_dh_params_file = zstrdup(o->ptr); + } config_set_special_field("tls-ca-cert-file") { + if (tlsConfigure(server.tls_cert_file, server.tls_key_file, + server.tls_dh_params_file, (char *) o->ptr) == C_ERR) { + addReplyError(c, + "Unable to configure tls-ca-cert-file. Check server logs."); + return; + } + zfree(server.tls_ca_cert_file); + server.tls_ca_cert_file = zstrdup(o->ptr); + } config_set_bool_field("tls-auth-clients", server.tls_auth_clients) { + /* Everyhing else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -1307,6 +1373,10 @@ void configGetCommand(client *c) { config_get_string_field("pidfile",server.pidfile); config_get_string_field("slave-announce-ip",server.slave_announce_ip); config_get_string_field("replica-announce-ip",server.slave_announce_ip); + config_get_string_field("tls-cert-file",server.tls_cert_file); + config_get_string_field("tls-key-file",server.tls_key_file); + config_get_string_field("tls-dh-params-file",server.tls_dh_params_file); + config_get_string_field("tls-ca-cert-file",server.tls_ca_cert_file); /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); @@ -1354,6 +1424,7 @@ void configGetCommand(client *c) { config_get_numerical_field("slowlog-max-len", server.slowlog_max_len); config_get_numerical_field("tracking-table-max-fill", server.tracking_table_max_fill); config_get_numerical_field("port",server.port); + config_get_numerical_field("tls-port",server.tls_port); config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); config_get_numerical_field("tcp-backlog",server.tcp_backlog); @@ -1393,6 +1464,9 @@ void configGetCommand(client *c) { } config_get_bool_field("activedefrag", server.active_defrag_enabled); + config_get_bool_field("tls-cluster",server.tls_cluster); + config_get_bool_field("tls-replication",server.tls_replication); + config_get_bool_field("tls-auth-clients",server.tls_auth_clients); /* Enum values */ config_get_enum_field("maxmemory-policy", @@ -2113,10 +2187,13 @@ int rewriteConfig(char *path) { } rewriteConfigStringOption(state,"pidfile",server.pidfile,CONFIG_DEFAULT_PID_FILE); - rewriteConfigNumericalOption(state,"port",server.port,CONFIG_DEFAULT_SERVER_PORT); + rewriteConfigNumericalOption(state,"tls-port",server.tls_port,CONFIG_DEFAULT_SERVER_TLS_PORT); rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); + rewriteConfigYesNoOption(state,"tls-cluster",server.tls_cluster,0); + rewriteConfigYesNoOption(state,"tls-replication",server.tls_replication,0); + rewriteConfigYesNoOption(state,"tls-auth-clients",server.tls_auth_clients,1); rewriteConfigBindOption(state); rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); @@ -2195,6 +2272,10 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); rewriteConfigNumericalOption(state,"rdb-key-save-delay",server.rdb_key_save_delay,CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY); + rewriteConfigStringOption(state,"tls-cert-file",server.tls_cert_file,NULL); + rewriteConfigStringOption(state,"tls-key-file",server.tls_key_file,NULL); + rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_dh_params_file,NULL); + rewriteConfigStringOption(state,"tls-ca-cert-file",server.tls_ca_cert_file,NULL); /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); diff --git a/src/connection.c b/src/connection.c new file mode 100644 index 000000000..62c6f3506 --- /dev/null +++ b/src/connection.c @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "server.h" +#include "connhelpers.h" + +/* The connections module provides a lean abstraction of network connections + * to avoid direct socket and async event management across the Redis code base. + * + * It does NOT provide advanced connection features commonly found in similar + * libraries such as complete in/out buffer management, throttling, etc. These + * functions remain in networking.c. + * + * The primary goal is to allow transparent handling of TCP and TLS based + * connections. To do so, connections have the following properties: + * + * 1. A connection may live before its corresponding socket exists. This + * allows various context and configuration setting to be handled before + * establishing the actual connection. + * 2. The caller may register/unregister logical read/write handlers to be + * called when the connection has data to read from/can accept writes. + * These logical handlers may or may not correspond to actual AE events, + * depending on the implementation (for TCP they are; for TLS they aren't). + */ + +ConnectionType CT_Socket; + +/* When a connection is created we must know its type already, but the + * underlying socket may or may not exist: + * + * - For accepted connections, it exists as we do not model the listen/accept + * part; So caller calls connCreateSocket() followed by connAccept(). + * - For outgoing connections, the socket is created by the connection module + * itself; So caller calls connCreateSocket() followed by connConnect(), + * which registers a connect callback that fires on connected/error state + * (and after any transport level handshake was done). + * + * NOTE: An earlier version relied on connections being part of other structs + * and not independently allocated. This could lead to further optimizations + * like using container_of(), etc. However it was discontinued in favor of + * this approach for these reasons: + * + * 1. In some cases conns are created/handled outside the context of the + * containing struct, in which case it gets a bit awkward to copy them. + * 2. Future implementations may wish to allocate arbitrary data for the + * connection. + * 3. The container_of() approach is anyway risky because connections may + * be embedded in different structs, not just client. + */ + +connection *connCreateSocket() { + connection *conn = zcalloc(sizeof(connection)); + conn->type = &CT_Socket; + conn->fd = -1; + + return conn; +} + +/* Create a new socket-type connection that is already associated with + * an accepted connection. + * + * The socket is not read for I/O until connAccept() was called and + * invoked the connection-level accept handler. + */ +connection *connCreateAcceptedSocket(int fd) { + connection *conn = connCreateSocket(); + conn->fd = fd; + conn->state = CONN_STATE_ACCEPTING; + return conn; +} + +static int connSocketConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + int fd = anetTcpNonBlockBestEffortBindConnect(NULL,addr,port,src_addr); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTING; + + conn->conn_handler = connect_handler; + aeCreateFileEvent(server.el, conn->fd, AE_WRITABLE, + conn->type->ae_handler, conn); + + return C_OK; +} + +/* Returns true if a write handler is registered */ +int connHasWriteHandler(connection *conn) { + return conn->write_handler != NULL; +} + +/* Returns true if a read handler is registered */ +int connHasReadHandler(connection *conn) { + return conn->read_handler != NULL; +} + +/* Associate a private data pointer with the connection */ +void connSetPrivateData(connection *conn, void *data) { + conn->private_data = data; +} + +/* Get the associated private data pointer */ +void *connGetPrivateData(connection *conn) { + return conn->private_data; +} + +/* ------ Pure socket connections ------- */ + +/* A very incomplete list of implementation-specific calls. Much of the above shall + * move here as we implement additional connection types. + */ + +static int connSocketShutdown(connection *conn, int how) { + return shutdown(conn->fd, how); +} + +/* Close the connection and free resources. */ +static void connSocketClose(connection *conn) { + if (conn->fd != -1) { + aeDeleteFileEvent(server.el,conn->fd,AE_READABLE); + aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + close(conn->fd); + conn->fd = -1; + } + + /* If called from within a handler, schedule the close but + * keep the connection until the handler returns. + */ + if (conn->flags & CONN_FLAG_IN_HANDLER) { + conn->flags |= CONN_FLAG_CLOSE_SCHEDULED; + return; + } + + zfree(conn); +} + +static int connSocketWrite(connection *conn, const void *data, size_t data_len) { + int ret = write(conn->fd, data, data_len); + if (!ret) { + conn->state = CONN_STATE_CLOSED; + } else if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketRead(connection *conn, void *buf, size_t buf_len) { + int ret = read(conn->fd, buf, buf_len); + if (!ret) { + conn->state = CONN_STATE_CLOSED; + } else if (ret < 0 && errno != EAGAIN) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } + + return ret; +} + +static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + if (conn->state != CONN_STATE_ACCEPTING) return C_ERR; + conn->state = CONN_STATE_CONNECTED; + if (!callHandler(conn, accept_handler)) return C_ERR; + return C_OK; +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + */ +static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { + if (func == conn->write_handler) return C_OK; + + conn->write_handler = func; + if (!conn->write_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + else + if (aeCreateFileEvent(server.el,conn->fd,AE_WRITABLE, + conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static int connSocketSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + if (func == conn->read_handler) return C_OK; + + conn->read_handler = func; + if (!conn->read_handler) + aeDeleteFileEvent(server.el,conn->fd,AE_READABLE); + else + if (aeCreateFileEvent(server.el,conn->fd, + AE_READABLE,conn->type->ae_handler,conn) == AE_ERR) return C_ERR; + return C_OK; +} + +static const char *connSocketGetLastError(connection *conn) { + return strerror(conn->last_errno); +} + +static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) +{ + UNUSED(el); + UNUSED(fd); + connection *conn = clientData; + + if (conn->state == CONN_STATE_CONNECTING && + (mask & AE_WRITABLE) && conn->conn_handler) { + + if (connGetSocketError(conn)) { + conn->last_errno = errno; + conn->state = CONN_STATE_ERROR; + } else { + conn->state = CONN_STATE_CONNECTED; + } + + if (!conn->write_handler) aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); + + if (!callHandler(conn, conn->conn_handler)) return; + conn->conn_handler = NULL; + } + + /* Handle normal I/O flows */ + if ((mask & AE_READABLE) && conn->read_handler) { + if (!callHandler(conn, conn->read_handler)) return; + } + if ((mask & AE_WRITABLE) && conn->write_handler) { + if (!callHandler(conn, conn->write_handler)) return; + } +} + +static int connSocketBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + int fd = anetTcpNonBlockConnect(NULL,addr,port); + if (fd == -1) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = errno; + return C_ERR; + } + + if ((aeWait(fd, AE_WRITABLE, timeout) & AE_WRITABLE) == 0) { + conn->state = CONN_STATE_ERROR; + conn->last_errno = ETIMEDOUT; + } + + conn->fd = fd; + conn->state = CONN_STATE_CONNECTED; + return C_OK; +} + +/* Connection-based versions of syncio.c functions. + * NOTE: This should ideally be refactored out in favor of pure async work. + */ + +static ssize_t connSocketSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncWrite(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncRead(conn->fd, ptr, size, timeout); +} + +static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return syncReadLine(conn->fd, ptr, size, timeout); +} + + +ConnectionType CT_Socket = { + .ae_handler = connSocketEventHandler, + .close = connSocketClose, + .shutdown = connSocketShutdown, + .write = connSocketWrite, + .read = connSocketRead, + .accept = connSocketAccept, + .connect = connSocketConnect, + .set_write_handler = connSocketSetWriteHandler, + .set_read_handler = connSocketSetReadHandler, + .get_last_error = connSocketGetLastError, + .blocking_connect = connSocketBlockingConnect, + .sync_write = connSocketSyncWrite, + .sync_read = connSocketSyncRead, + .sync_readline = connSocketSyncReadLine +}; + + +int connGetSocketError(connection *conn) { + int sockerr = 0; + socklen_t errlen = sizeof(sockerr); + + if (getsockopt(conn->fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) + sockerr = errno; + return sockerr; +} + +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port) { + return anetPeerToString(conn ? conn->fd : -1, ip, ip_len, port); +} + +int connFormatPeer(connection *conn, char *buf, size_t buf_len) { + return anetFormatPeer(conn ? conn->fd : -1, buf, buf_len); +} + +int connSockName(connection *conn, char *ip, size_t ip_len, int *port) { + return anetSockName(conn->fd, ip, ip_len, port); +} + +int connBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetBlock(NULL, conn->fd); +} + +int connNonBlock(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetNonBlock(NULL, conn->fd); +} + +int connEnableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetEnableTcpNoDelay(NULL, conn->fd); +} + +int connDisableTcpNoDelay(connection *conn) { + if (conn->fd == -1) return C_ERR; + return anetDisableTcpNoDelay(NULL, conn->fd); +} + +int connKeepAlive(connection *conn, int interval) { + if (conn->fd == -1) return C_ERR; + return anetKeepAlive(NULL, conn->fd, interval); +} + +int connSendTimeout(connection *conn, long long ms) { + return anetSendTimeout(NULL, conn->fd, ms); +} + +int connRecvTimeout(connection *conn, long long ms) { + return anetRecvTimeout(NULL, conn->fd, ms); +} + +int connGetState(connection *conn) { + return conn->state; +} + +/* Return a text that describes the connection, suitable for inclusion + * in CLIENT LIST and similar outputs. + * + * For sockets, we always return "fd=" to maintain compatibility. + */ +const char *connGetInfo(connection *conn, char *buf, size_t buf_len) { + snprintf(buf, buf_len-1, "fd=%i", conn->fd); + return buf; +} + diff --git a/src/connection.h b/src/connection.h new file mode 100644 index 000000000..e3e844f95 --- /dev/null +++ b/src/connection.h @@ -0,0 +1,211 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNECTION_H +#define __REDIS_CONNECTION_H + +#define CONN_INFO_LEN 32 + +struct aeEventLoop; +typedef struct connection connection; + +typedef enum { + CONN_STATE_NONE = 0, + CONN_STATE_CONNECTING, + CONN_STATE_ACCEPTING, + CONN_STATE_CONNECTED, + CONN_STATE_CLOSED, + CONN_STATE_ERROR +} ConnectionState; + +#define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */ +#define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */ + +typedef void (*ConnectionCallbackFunc)(struct connection *conn); + +typedef struct ConnectionType { + void (*ae_handler)(struct aeEventLoop *el, int fd, void *clientData, int mask); + int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); + int (*write)(struct connection *conn, const void *data, size_t data_len); + int (*read)(struct connection *conn, void *buf, size_t buf_len); + int (*shutdown)(struct connection *conn, int how); + void (*close)(struct connection *conn); + int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); + int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler); + int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler); + const char *(*get_last_error)(struct connection *conn); + int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout); + ssize_t (*sync_write)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_read)(struct connection *conn, char *ptr, ssize_t size, long long timeout); + ssize_t (*sync_readline)(struct connection *conn, char *ptr, ssize_t size, long long timeout); +} ConnectionType; + +struct connection { + ConnectionType *type; + ConnectionState state; + int flags; + int last_errno; + void *private_data; + ConnectionCallbackFunc conn_handler; + ConnectionCallbackFunc write_handler; + ConnectionCallbackFunc read_handler; + int fd; +}; + +/* The connection module does not deal with listening and accepting sockets, + * so we assume we have a socket when an incoming connection is created. + * + * The fd supplied should therefore be associated with an already accept()ed + * socket. + * + * connAccept() may directly call accept_handler(), or return and call it + * at a later time. This behavior is a bit awkward but aims to reduce the need + * to wait for the next event loop, if no additional handshake is required. + */ + +static inline int connAccept(connection *conn, ConnectionCallbackFunc accept_handler) { + return conn->type->accept(conn, accept_handler); +} + +/* Establish a connection. The connect_handler will be called when the connection + * is established, or if an error has occured. + * + * The connection handler will be responsible to set up any read/write handlers + * as needed. + * + * If C_ERR is returned, the operation failed and the connection handler shall + * not be expected. + */ +static inline int connConnect(connection *conn, const char *addr, int port, const char *src_addr, + ConnectionCallbackFunc connect_handler) { + return conn->type->connect(conn, addr, port, src_addr, connect_handler); +} + +/* Blocking connect. + * + * NOTE: This is implemented in order to simplify the transition to the abstract + * connections, but should probably be refactored out of cluster.c and replication.c, + * in favor of a pure async implementation. + */ +static inline int connBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { + return conn->type->blocking_connect(conn, addr, port, timeout); +} + +/* Write to connection, behaves the same as write(2). + * + * Like write(2), a short write is possible. A -1 return indicates an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connWrite(connection *conn, const void *data, size_t data_len) { + return conn->type->write(conn, data, data_len); +} + +/* Read from the connection, behaves the same as read(2). + * + * Like read(2), a short read is possible. A return value of 0 will indicate the + * connection was closed, and -1 will indicate an error. + * + * The caller should NOT rely on errno. Testing for an EAGAIN-like condition, use + * connGetState() to see if the connection state is still CONN_STATE_CONNECTED. + */ +static inline int connRead(connection *conn, void *buf, size_t buf_len) { + return conn->type->read(conn, buf, buf_len); +} + +/* Register a write handler, to be called when the connection is writable. + * If NULL, the existing handler is removed. + */ +static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_write_handler(conn, func); +} + +/* Register a read handler, to be called when the connection is readable. + * If NULL, the existing handler is removed. + */ +static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + return conn->type->set_read_handler(conn, func); +} + +static inline void connClose(connection *conn) { + conn->type->close(conn); +} + +static inline int connShutdown(connection *conn, int how) { + return conn->type->shutdown(conn, how); +} + +/* Returns the last error encountered by the connection, as a string. If no error, + * a NULL is returned. + */ +static inline const char *connGetLastError(connection *conn) { + return conn->type->get_last_error(conn); +} + +static inline ssize_t connSyncWrite(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_write(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncRead(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_read(conn, ptr, size, timeout); +} + +static inline ssize_t connSyncReadLine(connection *conn, char *ptr, ssize_t size, long long timeout) { + return conn->type->sync_readline(conn, ptr, size, timeout); +} + +connection *connCreateSocket(); +connection *connCreateAcceptedSocket(int fd); + +connection *connCreateTLS(); +connection *connCreateAcceptedTLS(int fd, int require_auth); + +void connSetPrivateData(connection *conn, void *data); +void *connGetPrivateData(connection *conn); +int connGetState(connection *conn); +int connHasWriteHandler(connection *conn); +int connHasReadHandler(connection *conn); +int connGetSocketError(connection *conn); + +/* anet-style wrappers to conns */ +int connBlock(connection *conn); +int connNonBlock(connection *conn); +int connEnableTcpNoDelay(connection *conn); +int connDisableTcpNoDelay(connection *conn); +int connKeepAlive(connection *conn, int interval); +int connSendTimeout(connection *conn, long long ms); +int connRecvTimeout(connection *conn, long long ms); +int connPeerToString(connection *conn, char *ip, size_t ip_len, int *port); +int connFormatPeer(connection *conn, char *buf, size_t buf_len); +int connSockName(connection *conn, char *ip, size_t ip_len, int *port); +const char *connGetInfo(connection *conn, char *buf, size_t buf_len); + +#endif /* __REDIS_CONNECTION_H */ diff --git a/src/connhelpers.h b/src/connhelpers.h new file mode 100644 index 000000000..2ceccd085 --- /dev/null +++ b/src/connhelpers.h @@ -0,0 +1,60 @@ + +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_CONNHELPERS_H +#define __REDIS_CONNHELPERS_H + +#include "connection.h" + +static inline void enterHandler(connection *conn) { + conn->flags |= CONN_FLAG_IN_HANDLER; +} + +static inline int exitHandler(connection *conn) { + conn->flags &= ~CONN_FLAG_IN_HANDLER; + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + connClose(conn); + return 0; + } + return 1; +} + +static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { + conn->flags |= CONN_FLAG_IN_HANDLER; + if (handler) handler(conn); + conn->flags &= ~CONN_FLAG_IN_HANDLER; + if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { + connClose(conn); + return 0; + } + return 1; +} + +#endif /* __REDIS_CONNHELPERS_H */ diff --git a/src/debug.c b/src/debug.c index 1f1157d4a..a2d61d8ab 100644 --- a/src/debug.c +++ b/src/debug.c @@ -699,11 +699,12 @@ void _serverAssert(const char *estr, const char *file, int line) { void _serverAssertPrintClientInfo(const client *c) { int j; + char conninfo[CONN_INFO_LEN]; bugReportStart(); serverLog(LL_WARNING,"=== ASSERTION FAILED CLIENT CONTEXT ==="); - serverLog(LL_WARNING,"client->flags = %llu", (unsigned long long)c->flags); - serverLog(LL_WARNING,"client->fd = %d", c->fd); + serverLog(LL_WARNING,"client->flags = %llu", (unsigned long long) c->flags); + serverLog(LL_WARNING,"client->conn = %s", connGetInfo(c->conn, conninfo, sizeof(conninfo))); serverLog(LL_WARNING,"client->argc = %d", c->argc); for (j=0; j < c->argc; j++) { char buf[128]; diff --git a/src/module.c b/src/module.c index ab614c529..8669586f4 100644 --- a/src/module.c +++ b/src/module.c @@ -2764,7 +2764,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* Create the client and dispatch the command. */ va_start(ap, fmt); - c = createClient(-1); + c = createClient(NULL); c->user = NULL; /* Root user. */ argv = moduleCreateArgvFromUserFormat(cmdname,fmt,&argc,&flags,ap); replicate = flags & REDISMODULE_ARGV_REPLICATE; @@ -3681,7 +3681,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ bc->free_privdata = free_privdata; bc->privdata = NULL; - bc->reply_client = createClient(-1); + bc->reply_client = createClient(NULL); bc->reply_client->flags |= CLIENT_MODULE; bc->dbid = c->db->id; c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0; @@ -3922,7 +3922,7 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { * access it safely from another thread, so we create a fake client here * in order to keep things like the currently selected database and similar * things. */ - ctx->client = createClient(-1); + ctx->client = createClient(NULL); if (bc) { selectDb(ctx->client,bc->dbid); ctx->client->id = bc->client->id; @@ -5113,7 +5113,7 @@ void moduleInitModulesSystem(void) { /* Set up the keyspace notification susbscriber list and static client */ moduleKeyspaceSubscribers = listCreate(); - moduleFreeContextReusedClient = createClient(-1); + moduleFreeContextReusedClient = createClient(NULL); moduleFreeContextReusedClient->flags |= CLIENT_MODULE; moduleFreeContextReusedClient->user = NULL; /* root user. */ diff --git a/src/networking.c b/src/networking.c index 7555ca77d..2d00b0e74 100644 --- a/src/networking.c +++ b/src/networking.c @@ -84,32 +84,27 @@ void linkClient(client *c) { raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL); } -client *createClient(int fd) { +client *createClient(connection *conn) { client *c = zmalloc(sizeof(client)); - /* passing -1 as fd it is possible to create a non connected client. + /* passing NULL as conn it is possible to create a non connected client. * This is useful since all the commands needs to be executed * in the context of a client. When commands are executed in other * contexts (for instance a Lua script) we need a non connected client. */ - if (fd != -1) { - anetNonBlock(NULL,fd); - anetEnableTcpNoDelay(NULL,fd); + if (conn) { + connNonBlock(conn); + connEnableTcpNoDelay(conn); if (server.tcpkeepalive) - anetKeepAlive(NULL,fd,server.tcpkeepalive); - if (aeCreateFileEvent(server.el,fd,AE_READABLE, - readQueryFromClient, c) == AE_ERR) - { - close(fd); - zfree(c); - return NULL; - } + connKeepAlive(conn,server.tcpkeepalive); + connSetReadHandler(conn, readQueryFromClient); + connSetPrivateData(conn, c); } selectDb(c,0); uint64_t client_id = ++server.next_client_id; c->id = client_id; c->resp = 2; - c->fd = fd; + c->conn = conn; c->name = NULL; c->bufpos = 0; c->qb_pos = 0; @@ -161,7 +156,7 @@ client *createClient(int fd) { c->client_tracking_redirection = 0; listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); - if (fd != -1) linkClient(c); + if (conn) linkClient(c); initClientMultiState(c); return c; } @@ -227,7 +222,7 @@ int prepareClientToWrite(client *c) { if ((c->flags & CLIENT_MASTER) && !(c->flags & CLIENT_MASTER_FORCE_REPLY)) return C_ERR; - if (c->fd <= 0) return C_ERR; /* Fake client for AOF loading. */ + if (!c->conn) return C_ERR; /* Fake client for AOF loading. */ /* Schedule the client to write the output buffers to the socket, unless * it should already be setup to do so (it has already pending data). */ @@ -777,28 +772,13 @@ int clientHasPendingReplies(client *c) { return c->bufpos || listLength(c->reply); } -#define MAX_ACCEPTS_PER_CALL 1000 -static void acceptCommonHandler(int fd, int flags, char *ip) { - client *c; - if ((c = createClient(fd)) == NULL) { - serverLog(LL_WARNING, - "Error registering fd event for the new client: %s (fd=%d)", - strerror(errno),fd); - close(fd); /* May be already closed, just ignore errors */ - return; - } - /* If maxclient directive is set and this is one client more... close the - * connection. Note that we create the client instead to check before - * for this condition, since now the socket is already set in non-blocking - * mode and we can send an error for free using the Kernel I/O */ - if (listLength(server.clients) > server.maxclients) { - char *err = "-ERR max number of clients reached\r\n"; +void clientAcceptHandler(connection *conn) { + client *c = connGetPrivateData(conn); - /* That's a best effort error message, don't check write errors */ - if (write(c->fd,err,strlen(err)) == -1) { - /* Nothing to do, Just to avoid the warning... */ - } - server.stat_rejected_conn++; + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_WARNING, + "Error accepting a client connection: %s", + connGetLastError(conn)); freeClient(c); return; } @@ -810,10 +790,12 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { if (server.protected_mode && server.bindaddr_count == 0 && DefaultUser->flags & USER_FLAG_NOPASS && - !(flags & CLIENT_UNIX_SOCKET) && - ip != NULL) + !(c->flags & CLIENT_UNIX_SOCKET)) { - if (strcmp(ip,"127.0.0.1") && strcmp(ip,"::1")) { + char cip[NET_IP_STR_LEN+1] = { 0 }; + connPeerToString(conn, cip, sizeof(cip)-1, NULL); + + if (strcmp(cip,"127.0.0.1") && strcmp(cip,"::1")) { char *err = "-DENIED Redis is running in protected mode because protected " "mode is enabled, no bind address was specified, no " @@ -835,7 +817,7 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { "4) Setup a bind address or an authentication password. " "NOTE: You only need to do one of the above things in order for " "the server to start accepting connections from the outside.\r\n"; - if (write(c->fd,err,strlen(err)) == -1) { + if (connWrite(c->conn,err,strlen(err)) == -1) { /* Nothing to do, Just to avoid the warning... */ } server.stat_rejected_conn++; @@ -845,7 +827,63 @@ static void acceptCommonHandler(int fd, int flags, char *ip) { } server.stat_numconnections++; +} + + +#define MAX_ACCEPTS_PER_CALL 1000 +static void acceptCommonHandler(connection *conn, int flags, char *ip) { + client *c; + UNUSED(ip); + + /* Admission control will happen before a client is created and connAccept() + * called, because we don't want to even start transport-level negotiation + * if rejected. + */ + if (listLength(server.clients) >= server.maxclients) { + char *err = "-ERR max number of clients reached\r\n"; + + /* That's a best effort error message, don't check write errors. + * Note that for TLS connections, no handshake was done yet so nothing is written + * and the connection will just drop. + */ + if (connWrite(conn,err,strlen(err)) == -1) { + /* Nothing to do, Just to avoid the warning... */ + } + server.stat_rejected_conn++; + connClose(conn); + return; + } + + /* Create connection and client */ + if ((c = createClient(conn)) == NULL) { + char conninfo[100]; + serverLog(LL_WARNING, + "Error registering fd event for the new client: %s (conn: %s)", + connGetLastError(conn), + connGetInfo(conn, conninfo, sizeof(conninfo))); + connClose(conn); /* May be already closed, just ignore errors */ + return; + } + + /* Last chance to keep flags */ c->flags |= flags; + + /* Initiate accept. + * + * Note that connAccept() is free to do two things here: + * 1. Call clientAcceptHandler() immediately; + * 2. Schedule a future call to clientAcceptHandler(). + * + * Because of that, we must do nothing else afterwards. + */ + if (connAccept(conn, clientAcceptHandler) == C_ERR) { + char conninfo[100]; + serverLog(LL_WARNING, + "Error accepting a client connection: %s (conn: %s)", + connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); + connClose(conn); + return; + } } void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -864,7 +902,27 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { return; } serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); - acceptCommonHandler(cfd,0,cip); + acceptCommonHandler(connCreateAcceptedSocket(cfd),0,cip); + } +} + +void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask) { + int cport, cfd, max = MAX_ACCEPTS_PER_CALL; + char cip[NET_IP_STR_LEN]; + UNUSED(el); + UNUSED(mask); + UNUSED(privdata); + + while(max--) { + cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport); + if (cfd == ANET_ERR) { + if (errno != EWOULDBLOCK) + serverLog(LL_WARNING, + "Accepting client connection: %s", server.neterr); + return; + } + serverLog(LL_VERBOSE,"Accepted %s:%d", cip, cport); + acceptCommonHandler(connCreateAcceptedTLS(cfd, server.tls_auth_clients),0,cip); } } @@ -883,7 +941,7 @@ void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask) { return; } serverLog(LL_VERBOSE,"Accepted connection to %s", server.unixsocket); - acceptCommonHandler(cfd,CLIENT_UNIX_SOCKET,NULL); + acceptCommonHandler(connCreateAcceptedSocket(cfd),CLIENT_UNIX_SOCKET,NULL); } } @@ -914,10 +972,10 @@ void unlinkClient(client *c) { /* If this is marked as current client unset it. */ if (server.current_client == c) server.current_client = NULL; - /* Certain operations must be done only if the client has an active socket. + /* Certain operations must be done only if the client has an active connection. * If the client was already unlinked or if it's a "fake client" the - * fd is already set to -1. */ - if (c->fd != -1) { + * conn is already set to NULL. */ + if (c->conn) { /* Remove from the list of active clients. */ if (c->client_list_node) { uint64_t id = htonu64(c->id); @@ -931,16 +989,11 @@ void unlinkClient(client *c) { * shutdown the socket the fork will continue to write to the slave * and the salve will only find out that it was disconnected when * it will finish reading the rdb. */ - if ((c->flags & CLIENT_SLAVE) && - (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)) { - shutdown(c->fd, SHUT_RDWR); - } - - /* Unregister async I/O handlers and close the socket. */ - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); - close(c->fd); - c->fd = -1; + int need_shutdown = ((c->flags & CLIENT_SLAVE) && + (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)); + if (need_shutdown) connShutdown(c->conn, SHUT_RDWR); + connClose(c->conn); + c->conn = NULL; } /* Remove from the list of pending writes if needed. */ @@ -1112,19 +1165,20 @@ client *lookupClientByID(uint64_t id) { /* Write data in output buffers to client. Return C_OK if the client * is still valid after the call, C_ERR if it was freed because of some - * error. + * error. If handler_installed is set, it will attempt to clear the + * write event. * * This function is called by threads, but always with handler_installed * set to 0. So when handler_installed is set to 0 the function must be * thread safe. */ -int writeToClient(int fd, client *c, int handler_installed) { +int writeToClient(client *c, int handler_installed) { ssize_t nwritten = 0, totwritten = 0; size_t objlen; clientReplyBlock *o; while(clientHasPendingReplies(c)) { if (c->bufpos > 0) { - nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); + nwritten = connWrite(c->conn,c->buf+c->sentlen,c->bufpos-c->sentlen); if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1145,7 +1199,7 @@ int writeToClient(int fd, client *c, int handler_installed) { continue; } - nwritten = write(fd, o->buf + c->sentlen, objlen - c->sentlen); + nwritten = connWrite(c->conn, o->buf + c->sentlen, objlen - c->sentlen); if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -1180,11 +1234,11 @@ int writeToClient(int fd, client *c, int handler_installed) { } server.stat_net_output_bytes += totwritten; if (nwritten == -1) { - if (errno == EAGAIN) { + if (connGetState(c->conn) == CONN_STATE_CONNECTED) { nwritten = 0; } else { serverLog(LL_VERBOSE, - "Error writing to client: %s", strerror(errno)); + "Error writing to client: %s", connGetLastError(c->conn)); freeClientAsync(c); return C_ERR; } @@ -1202,7 +1256,7 @@ int writeToClient(int fd, client *c, int handler_installed) { * adDeleteFileEvent() is not thread safe: however writeToClient() * is always called with handler_installed set to 0 from threads * so we are fine. */ - if (handler_installed) aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + if (handler_installed) connSetWriteHandler(c->conn, NULL); /* Close connection after entire reply has been sent. */ if (c->flags & CLIENT_CLOSE_AFTER_REPLY) { @@ -1214,10 +1268,9 @@ int writeToClient(int fd, client *c, int handler_installed) { } /* Write event handler. Just send data to the client. */ -void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { - UNUSED(el); - UNUSED(mask); - writeToClient(fd,privdata,1); +void sendReplyToClient(connection *conn) { + client *c = connGetPrivateData(conn); + writeToClient(c,1); } /* This function is called just before entering the event loop, in the hope @@ -1240,7 +1293,7 @@ int handleClientsWithPendingWrites(void) { if (c->flags & CLIENT_PROTECTED) continue; /* Try to write buffers to the client socket. */ - if (writeToClient(c->fd,c,0) == C_ERR) continue; + if (writeToClient(c,0) == C_ERR) continue; /* If after the synchronous writes above we still have data to * output to the client, we need to install the writable handler. */ @@ -1256,10 +1309,9 @@ int handleClientsWithPendingWrites(void) { { ae_flags |= AE_BARRIER; } - if (aeCreateFileEvent(server.el, c->fd, ae_flags, - sendReplyToClient, c) == AE_ERR) - { - freeClientAsync(c); + /* TODO: Handle write barriers in connection */ + if (connSetWriteHandler(c->conn, sendReplyToClient) == C_ERR) { + freeClientAsync(c); } } } @@ -1305,15 +1357,15 @@ void resetClient(client *c) { * path, it is not really released, but only marked for later release. */ void protectClient(client *c) { c->flags |= CLIENT_PROTECTED; - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + connSetReadHandler(c->conn,NULL); + connSetWriteHandler(c->conn,NULL); } /* This will undo the client protection done by protectClient() */ void unprotectClient(client *c) { if (c->flags & CLIENT_PROTECTED) { c->flags &= ~CLIENT_PROTECTED; - aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + connSetReadHandler(c->conn,readQueryFromClient); if (clientHasPendingReplies(c)) clientInstallWriteHandler(c); } } @@ -1710,12 +1762,10 @@ void processInputBufferAndReplicate(client *c) { } } -void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { - client *c = (client*) privdata; +void readQueryFromClient(connection *conn) { + client *c = connGetPrivateData(conn); int nread, readlen; size_t qblen; - UNUSED(el); - UNUSED(mask); /* Check if we want to read from the client later when exiting from * the event loop. This is the case if threaded I/O is enabled. */ @@ -1741,12 +1791,12 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { qblen = sdslen(c->querybuf); if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); - nread = read(fd, c->querybuf+qblen, readlen); + nread = connRead(c->conn, c->querybuf+qblen, readlen); if (nread == -1) { - if (errno == EAGAIN) { + if (connGetState(conn) == CONN_STATE_CONNECTED) { return; } else { - serverLog(LL_VERBOSE, "Reading from client: %s",strerror(errno)); + serverLog(LL_VERBOSE, "Reading from client: %s",connGetLastError(c->conn)); freeClientAsync(c); return; } @@ -1818,7 +1868,7 @@ void genClientPeerId(client *client, char *peerid, snprintf(peerid,peerid_len,"%s:0",server.unixsocket); } else { /* TCP client. */ - anetFormatPeer(client->fd,peerid,peerid_len); + connFormatPeer(client->conn,peerid,peerid_len); } } @@ -1839,8 +1889,7 @@ char *getClientPeerId(client *c) { /* Concatenate a string representing the state of a client in an human * readable format, into the sds string 's'. */ sds catClientInfoString(sds s, client *client) { - char flags[16], events[3], *p; - int emask; + char flags[16], events[3], conninfo[CONN_INFO_LEN], *p; p = flags; if (client->flags & CLIENT_SLAVE) { @@ -1864,16 +1913,17 @@ sds catClientInfoString(sds s, client *client) { if (p == flags) *p++ = 'N'; *p++ = '\0'; - emask = client->fd == -1 ? 0 : aeGetFileEvents(server.el,client->fd); p = events; - if (emask & AE_READABLE) *p++ = 'r'; - if (emask & AE_WRITABLE) *p++ = 'w'; + if (client->conn) { + if (connHasReadHandler(client->conn)) *p++ = 'r'; + if (connHasWriteHandler(client->conn)) *p++ = 'w'; + } *p = '\0'; return sdscatfmt(s, - "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s", + "id=%U addr=%s %s name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s user=%s", (unsigned long long) client->id, getClientPeerId(client), - client->fd, + connGetInfo(client->conn, conninfo, sizeof(conninfo)), client->name ? (char*)client->name->ptr : "", (long long)(server.unixtime - client->ctime), (long long)(server.unixtime - client->lastinteraction), @@ -2445,7 +2495,7 @@ int checkClientOutputBufferLimits(client *c) { * called from contexts where the client can't be freed safely, i.e. from the * lower level functions pushing data inside the client output buffers. */ void asyncCloseClientOnOutputBufferLimitReached(client *c) { - if (c->fd == -1) return; /* It is unsafe to free fake clients. */ + if (!c->conn) return; /* It is unsafe to free fake clients. */ serverAssert(c->reply_bytes < SIZE_MAX-(1024*64)); if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return; if (checkClientOutputBufferLimits(c)) { @@ -2468,8 +2518,7 @@ void flushSlavesOutputBuffers(void) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = listNodeValue(ln); - int events = aeGetFileEvents(server.el,slave->fd); - int can_receive_writes = (events & AE_WRITABLE) || + int can_receive_writes = connHasWriteHandler(slave->conn) || (slave->flags & CLIENT_PENDING_WRITE); /* We don't want to send the pending data to the replica in a few @@ -2491,7 +2540,7 @@ void flushSlavesOutputBuffers(void) { !slave->repl_put_online_on_ack && clientHasPendingReplies(slave)) { - writeToClient(slave->fd,slave,0); + writeToClient(slave,0); } } } @@ -2618,9 +2667,9 @@ void *IOThreadMain(void *myid) { while((ln = listNext(&li))) { client *c = listNodeValue(ln); if (io_threads_op == IO_THREADS_OP_WRITE) { - writeToClient(c->fd,c,0); + writeToClient(c,0); } else if (io_threads_op == IO_THREADS_OP_READ) { - readQueryFromClient(NULL,c->fd,c,0); + readQueryFromClient(c->conn); } else { serverPanic("io_threads_op value is unknown"); } @@ -2761,8 +2810,7 @@ int handleClientsWithPendingWritesUsingThreads(void) { /* Install the write handler if there are pending writes in some * of the clients. */ if (clientHasPendingReplies(c) && - aeCreateFileEvent(server.el, c->fd, AE_WRITABLE, - sendReplyToClient, c) == AE_ERR) + connSetWriteHandler(c->conn, sendReplyToClient) == AE_ERR) { freeClientAsync(c); } diff --git a/src/rdb.c b/src/rdb.c index e4dfc46b7..3b98f57bb 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2387,8 +2387,8 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { "Slave %s correctly received the streamed RDB file.", replicationGetSlaveName(slave)); /* Restore the socket as non-blocking. */ - anetNonBlock(NULL,slave->fd); - anetSendTimeout(NULL,slave->fd,0); + connNonBlock(slave->conn); + connSendTimeout(slave->conn,0); } } } @@ -2425,9 +2425,9 @@ void killRDBChild(void) { /* Spawn an RDB child that writes the RDB to the sockets of the slaves * that are currently in SLAVE_STATE_WAIT_BGSAVE_START state. */ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { - int *fds; + connection **conns; uint64_t *clientids; - int numfds; + int numconns; listNode *ln; listIter li; pid_t childpid; @@ -2445,26 +2445,26 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { /* Collect the file descriptors of the slaves we want to transfer * the RDB to, which are i WAIT_BGSAVE_START state. */ - fds = zmalloc(sizeof(int)*listLength(server.slaves)); + conns = zmalloc(sizeof(connection *)*listLength(server.slaves)); /* We also allocate an array of corresponding client IDs. This will * be useful for the child process in order to build the report * (sent via unix pipe) that will be sent to the parent. */ clientids = zmalloc(sizeof(uint64_t)*listLength(server.slaves)); - numfds = 0; + numconns = 0; listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = ln->value; if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) { - clientids[numfds] = slave->id; - fds[numfds++] = slave->fd; + clientids[numconns] = slave->id; + conns[numconns++] = slave->conn; replicationSetupSlaveForFullResync(slave,getPsyncInitialOffset()); /* Put the socket in blocking mode to simplify RDB transfer. * We'll restore it when the children returns (since duped socket * will share the O_NONBLOCK attribute with the parent). */ - anetBlock(NULL,slave->fd); - anetSendTimeout(NULL,slave->fd,server.repl_timeout*1000); + connBlock(slave->conn); + connSendTimeout(slave->conn,server.repl_timeout*1000); } } @@ -2476,8 +2476,8 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { int retval; rio slave_sockets; - rioInitWithFdset(&slave_sockets,fds,numfds); - zfree(fds); + rioInitWithConnset(&slave_sockets,conns,numconns); + zfree(conns); closeListeningSockets(0); redisSetProcTitle("redis-rdb-to-slaves"); @@ -2513,22 +2513,22 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { * can match the report with a specific slave, and 'error' is * set to 0 if the replication process terminated with a success * or the error code if an error occurred. */ - void *msg = zmalloc(sizeof(uint64_t)*(1+2*numfds)); + void *msg = zmalloc(sizeof(uint64_t)*(1+2*numconns)); uint64_t *len = msg; uint64_t *ids = len+1; int j, msglen; - *len = numfds; - for (j = 0; j < numfds; j++) { + *len = numconns; + for (j = 0; j < numconns; j++) { *ids++ = clientids[j]; - *ids++ = slave_sockets.io.fdset.state[j]; + *ids++ = slave_sockets.io.connset.state[j]; } /* Write the message to the parent. If we have no good slaves or * we are unable to transfer the message to the parent, we exit * with an error so that the parent will abort the replication * process with all the childre that were waiting. */ - msglen = sizeof(uint64_t)*(1+2*numfds); + msglen = sizeof(uint64_t)*(1+2*numconns); if (*len == 0 || write(server.rdb_pipe_write_result_to_parent,msg,msglen) != msglen) @@ -2538,7 +2538,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { zfree(msg); } zfree(clientids); - rioFreeFdset(&slave_sockets); + rioFreeConnset(&slave_sockets); exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ @@ -2554,7 +2554,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { client *slave = ln->value; int j; - for (j = 0; j < numfds; j++) { + for (j = 0; j < numconns; j++) { if (slave->id == clientids[j]) { slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START; break; @@ -2577,7 +2577,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { updateDictResizePolicy(); } zfree(clientids); - zfree(fds); + zfree(conns); return (childpid == -1) ? C_ERR : C_OK; } return C_OK; /* Unreached. */ diff --git a/src/redis-cli.c b/src/redis-cli.c index db53fc7d8..c7145927a 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -47,6 +47,9 @@ #include #include +#ifdef USE_OPENSSL +#include +#endif #include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ #include "dict.h" #include "adlist.h" @@ -188,6 +191,11 @@ static struct config { char *hostip; int hostport; char *hostsocket; + int tls; + char *sni; + char *cacert; + char *cert; + char *key; long repeat; long interval; int dbnum; @@ -751,6 +759,18 @@ static int cliSelect(void) { return REDIS_ERR; } +/* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if + * not building with TLS support. + */ +static int cliSecureConnection(redisContext *c) { +#ifdef USE_OPENSSL + return redisSecureConnection(c, config.cacert, config.cert, config.key, config.sni); +#else + (void) c; + return REDIS_OK; +#endif +} + /* Connect to the server. It is possible to pass certain flags to the function: * CC_FORCE: The connection is performed even if there is already * a connected socket. @@ -767,6 +787,16 @@ static int cliConnect(int flags) { context = redisConnectUnix(config.hostsocket); } + if (!context->err && config.tls) { + if (cliSecureConnection(context) == REDIS_ERR && !context->err) { + /* TODO: this check should be redundant, redis-cli should set err=1 */ + fprintf(stderr, "Could not negotiate a TLS connection.\n"); + context = NULL; + redisFree(context); + return REDIS_ERR; + } + } + if (context->err) { if (!(flags & CC_QUIET)) { fprintf(stderr,"Could not connect to Redis at "); @@ -782,6 +812,7 @@ static int cliConnect(int flags) { return REDIS_ERR; } + /* Set aggressive KEEP_ALIVE socket option in the Redis context socket * in order to prevent timeouts caused by the execution of long * commands. At the same time this improves the detection of real @@ -1245,6 +1276,9 @@ static redisReply *reconnectingRedisCommand(redisContext *c, const char *fmt, .. redisFree(c); c = redisConnect(config.hostip,config.hostport); + if (!c->err && config.tls) { + cliSecureConnection(c); + } usleep(1000000); } @@ -1434,6 +1468,18 @@ static int parseOptions(int argc, char **argv) { } else if (!strcmp(argv[i],"--cluster-search-multiple-owners")) { config.cluster_manager_command.flags |= CLUSTER_MANAGER_CMD_FLAG_CHECK_OWNERS; +#ifdef USE_OPENSSL + } else if (!strcmp(argv[i],"--tls")) { + config.tls = 1; + } else if (!strcmp(argv[i],"--sni")) { + config.sni = argv[++i]; + } else if (!strcmp(argv[i],"--cacert")) { + config.cacert = argv[++i]; + } else if (!strcmp(argv[i],"--cert")) { + config.cert = argv[++i]; + } else if (!strcmp(argv[i],"--key")) { + config.key = argv[++i]; +#endif } else if (!strcmp(argv[i],"-v") || !strcmp(argv[i], "--version")) { sds version = cliVersion(); printf("redis-cli %s\n", version); @@ -1522,6 +1568,12 @@ static void usage(void) { " -x Read last argument from STDIN.\n" " -d Multi-bulk delimiter in for raw formatting (default: \\n).\n" " -c Enable cluster mode (follow -ASK and -MOVED redirections).\n" +#ifdef USE_OPENSSL +" --tls Establish a secure TLS connection.\n" +" --cacert CA Certificate file to verify with.\n" +" --cert Client certificate to authenticate with.\n" +" --key Private key file to authenticate with.\n" +#endif " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty).\n" " --no-raw Force formatted output even when STDOUT is not a tty.\n" @@ -1544,7 +1596,9 @@ static void usage(void) { " --pipe Transfer raw Redis protocol from stdin to server.\n" " --pipe-timeout In --pipe mode, abort with error if after sending all data.\n" " no reply is received within seconds.\n" -" Default timeout: %d. Use 0 to wait forever.\n" +" Default timeout: %d. Use 0 to wait forever.\n", + version, REDIS_CLI_DEFAULT_PIPE_TIMEOUT); + fprintf(stderr, " --bigkeys Sample Redis keys looking for keys with many elements (complexity).\n" " --memkeys Sample Redis keys looking for keys consuming a lot of memory.\n" " --memkeys-samples Sample Redis keys looking for keys consuming a lot of memory.\n" @@ -1567,8 +1621,7 @@ static void usage(void) { " line interface.\n" " --help Output this help and exit.\n" " --version Output version and exit.\n" -"\n", - version, REDIS_CLI_DEFAULT_PIPE_TIMEOUT); +"\n"); /* Using another fprintf call to avoid -Woverlength-strings compile warning */ fprintf(stderr, "Cluster Manager Commands:\n" @@ -2336,6 +2389,9 @@ cleanup: static int clusterManagerNodeConnect(clusterManagerNode *node) { if (node->context) redisFree(node->context); node->context = redisConnect(node->ip, node->port); + if (!node->context->err && config.tls) { + cliSecureConnection(node->context); + } if (node->context->err) { fprintf(stderr,"Could not connect to Redis at "); fprintf(stderr,"%s:%d: %s\n", node->ip, node->port, diff --git a/src/replication.c b/src/replication.c index d6646c9ef..cbc52cb2e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -39,7 +39,7 @@ #include void replicationDiscardCachedMaster(void); -void replicationResurrectCachedMaster(int newfd); +void replicationResurrectCachedMaster(connection *conn); void replicationSendAck(void); void putSlaveOnline(client *slave); int cancelReplicationHandshake(void); @@ -57,7 +57,7 @@ char *replicationGetSlaveName(client *c) { ip[0] = '\0'; buf[0] = '\0'; if (c->slave_ip[0] != '\0' || - anetPeerToString(c->fd,ip,sizeof(ip),NULL) != -1) + connPeerToString(c->conn,ip,sizeof(ip),NULL) != -1) { /* Note that the 'ip' buffer is always larger than 'c->slave_ip' */ if (c->slave_ip[0] != '\0') memcpy(ip,c->slave_ip,sizeof(c->slave_ip)); @@ -432,7 +432,7 @@ int replicationSetupSlaveForFullResync(client *slave, long long offset) { if (!(slave->flags & CLIENT_PRE_PSYNC)) { buflen = snprintf(buf,sizeof(buf),"+FULLRESYNC %s %lld\r\n", server.replid,offset); - if (write(slave->fd,buf,buflen) != buflen) { + if (connWrite(slave->conn,buf,buflen) != buflen) { freeClientAsync(slave); return C_ERR; } @@ -519,7 +519,7 @@ int masterTryPartialResynchronization(client *c) { } else { buflen = snprintf(buf,sizeof(buf),"+CONTINUE\r\n"); } - if (write(c->fd,buf,buflen) != buflen) { + if (connWrite(c->conn,buf,buflen) != buflen) { freeClientAsync(c); return C_OK; } @@ -685,7 +685,7 @@ void syncCommand(client *c) { * paths will change the state if we handle the slave differently. */ c->replstate = SLAVE_STATE_WAIT_BGSAVE_START; if (server.repl_disable_tcp_nodelay) - anetDisableTcpNoDelay(NULL, c->fd); /* Non critical if it fails. */ + connDisableTcpNoDelay(c->conn); /* Non critical if it fails. */ c->repldbfd = -1; c->flags |= CLIENT_SLAVE; listAddNodeTail(server.slaves,c); @@ -862,8 +862,7 @@ void putSlaveOnline(client *slave) { slave->replstate = SLAVE_STATE_ONLINE; slave->repl_put_online_on_ack = 0; slave->repl_ack_time = server.unixtime; /* Prevent false timeout. */ - if (aeCreateFileEvent(server.el, slave->fd, AE_WRITABLE, - sendReplyToClient, slave) == AE_ERR) { + if (connSetWriteHandler(slave->conn, sendReplyToClient) == C_ERR) { serverLog(LL_WARNING,"Unable to register writable event for replica bulk transfer: %s", strerror(errno)); freeClient(slave); return; @@ -873,10 +872,8 @@ void putSlaveOnline(client *slave) { replicationGetSlaveName(slave)); } -void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { - client *slave = privdata; - UNUSED(el); - UNUSED(mask); +void sendBulkToSlave(connection *conn) { + client *slave = connGetPrivateData(conn); char buf[PROTO_IOBUF_LEN]; ssize_t nwritten, buflen; @@ -884,7 +881,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { * replication process. Currently the preamble is just the bulk count of * the file in the form "$\r\n". */ if (slave->replpreamble) { - nwritten = write(fd,slave->replpreamble,sdslen(slave->replpreamble)); + nwritten = connWrite(conn,slave->replpreamble,sdslen(slave->replpreamble)); if (nwritten == -1) { serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s", strerror(errno)); @@ -911,8 +908,8 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { freeClient(slave); return; } - if ((nwritten = write(fd,buf,buflen)) == -1) { - if (errno != EAGAIN) { + if ((nwritten = connWrite(conn,buf,buflen)) == -1) { + if (connGetState(conn) != CONN_STATE_CONNECTED) { serverLog(LL_WARNING,"Write error sending DB to replica: %s", strerror(errno)); freeClient(slave); @@ -924,7 +921,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { if (slave->repldboff == slave->repldbsize) { close(slave->repldbfd); slave->repldbfd = -1; - aeDeleteFileEvent(server.el,slave->fd,AE_WRITABLE); + connSetWriteHandler(slave->conn,NULL); putSlaveOnline(slave); } } @@ -1015,8 +1012,8 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type) { slave->replpreamble = sdscatprintf(sdsempty(),"$%lld\r\n", (unsigned long long) slave->repldbsize); - aeDeleteFileEvent(server.el,slave->fd,AE_WRITABLE); - if (aeCreateFileEvent(server.el, slave->fd, AE_WRITABLE, sendBulkToSlave, slave) == AE_ERR) { + connSetWriteHandler(slave->conn,NULL); + if (connSetWriteHandler(slave->conn,sendBulkToSlave) == C_ERR) { freeClient(slave); continue; } @@ -1084,9 +1081,8 @@ void replicationSendNewlineToMaster(void) { static time_t newline_sent; if (time(NULL) != newline_sent) { newline_sent = time(NULL); - if (write(server.repl_transfer_s,"\n",1) == -1) { - /* Pinging back in this stage is best-effort. */ - } + /* Pinging back in this stage is best-effort. */ + if (server.repl_transfer_s) connWrite(server.repl_transfer_s, "\n", 1); } } @@ -1100,8 +1096,10 @@ void replicationEmptyDbCallback(void *privdata) { /* Once we have a link with the master and the synchroniziation was * performed, this function materializes the master client we store * at server.master, starting from the specified file descriptor. */ -void replicationCreateMasterClient(int fd, int dbid) { - server.master = createClient(fd); +void replicationCreateMasterClient(connection *conn, int dbid) { + server.master = createClient(conn); + if (conn) + connSetReadHandler(server.master->conn, readQueryFromClient); server.master->flags |= CLIENT_MASTER; server.master->authenticated = 1; server.master->reploff = server.master_initial_offset; @@ -1189,7 +1187,7 @@ void disklessLoadRestoreBackups(redisDb *backup, int restore, int empty_db_flags /* Asynchronously read the SYNC payload we receive from a master */ #define REPL_MAX_WRITTEN_BEFORE_FSYNC (1024*1024*8) /* 8 MB */ -void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { +void readSyncBulkPayload(connection *conn) { char buf[4096]; ssize_t nread, readlen, nwritten; int use_diskless_load; @@ -1197,9 +1195,6 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { int empty_db_flags = server.repl_slave_lazy_flush ? EMPTYDB_ASYNC : EMPTYDB_NO_FLAGS; off_t left; - UNUSED(el); - UNUSED(privdata); - UNUSED(mask); /* Static vars used to hold the EOF mark, and the last bytes received * form the server: when they match, we reached the end of the transfer. */ @@ -1210,7 +1205,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { /* If repl_transfer_size == -1 we still have to read the bulk length * from the master reply. */ if (server.repl_transfer_size == -1) { - if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout*1000) == -1) { + if (connSyncReadLine(conn,buf,1024,server.repl_syncio_timeout*1000) == -1) { serverLog(LL_WARNING, "I/O error reading bulk count from MASTER: %s", strerror(errno)); @@ -1275,7 +1270,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { readlen = (left < (signed)sizeof(buf)) ? left : (signed)sizeof(buf); } - nread = read(fd,buf,readlen); + nread = connRead(conn,buf,readlen); if (nread <= 0) { serverLog(LL_WARNING,"I/O error trying to sync with MASTER: %s", (nread == -1) ? strerror(errno) : "connection lost"); @@ -1383,17 +1378,17 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { * handler, otherwise it will get called recursively since * rdbLoad() will call the event loop to process events from time to * time for non blocking loading. */ - aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE); + connSetReadHandler(conn, NULL); serverLog(LL_NOTICE, "MASTER <-> REPLICA sync: Loading DB in memory"); rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; if (use_diskless_load) { rio rdb; - rioInitWithFd(&rdb,fd,server.repl_transfer_size); + rioInitWithConn(&rdb,conn,server.repl_transfer_size); /* Put the socket in blocking mode to simplify RDB transfer. * We'll restore it when the RDB is received. */ - anetBlock(NULL,fd); - anetRecvTimeout(NULL,fd,server.repl_timeout*1000); + connBlock(conn); + connRecvTimeout(conn, server.repl_timeout*1000); startLoading(server.repl_transfer_size); if (rdbLoadRio(&rdb,&rsi,0) != C_OK) { @@ -1403,7 +1398,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { "Failed trying to load the MASTER synchronization DB " "from socket"); cancelReplicationHandshake(); - rioFreeFd(&rdb, NULL); + rioFreeConn(&rdb, NULL); if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { /* Restore the backed up databases. */ disklessLoadRestoreBackups(diskless_load_backup,1, @@ -1436,16 +1431,16 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { { serverLog(LL_WARNING,"Replication stream EOF marker is broken"); cancelReplicationHandshake(); - rioFreeFd(&rdb, NULL); + rioFreeConn(&rdb, NULL); return; } } /* Cleanup and restore the socket to the original state to continue * with the normal replication. */ - rioFreeFd(&rdb, NULL); - anetNonBlock(NULL,fd); - anetRecvTimeout(NULL,fd,0); + rioFreeConn(&rdb, NULL); + connNonBlock(conn); + connRecvTimeout(conn,0); } else { /* Ensure background save doesn't overwrite synced data */ if (server.rdb_child_pid != -1) { @@ -1522,7 +1517,7 @@ error: #define SYNC_CMD_READ (1<<0) #define SYNC_CMD_WRITE (1<<1) #define SYNC_CMD_FULL (SYNC_CMD_READ|SYNC_CMD_WRITE) -char *sendSynchronousCommand(int flags, int fd, ...) { +char *sendSynchronousCommand(int flags, connection *conn, ...) { /* Create the command to send to the master, we use redis binary * protocol to make sure correct arguments are sent. This function @@ -1533,7 +1528,7 @@ char *sendSynchronousCommand(int flags, int fd, ...) { sds cmd = sdsempty(); sds cmdargs = sdsempty(); size_t argslen = 0; - va_start(ap,fd); + va_start(ap,conn); while(1) { arg = va_arg(ap, char*); @@ -1550,12 +1545,12 @@ char *sendSynchronousCommand(int flags, int fd, ...) { sdsfree(cmdargs); /* Transfer command to the server. */ - if (syncWrite(fd,cmd,sdslen(cmd),server.repl_syncio_timeout*1000) + if (connSyncWrite(conn,cmd,sdslen(cmd),server.repl_syncio_timeout*1000) == -1) { sdsfree(cmd); return sdscatprintf(sdsempty(),"-Writing to master: %s", - strerror(errno)); + connGetLastError(conn)); } sdsfree(cmd); } @@ -1564,7 +1559,7 @@ char *sendSynchronousCommand(int flags, int fd, ...) { if (flags & SYNC_CMD_READ) { char buf[256]; - if (syncReadLine(fd,buf,sizeof(buf),server.repl_syncio_timeout*1000) + if (connSyncReadLine(conn,buf,sizeof(buf),server.repl_syncio_timeout*1000) == -1) { return sdscatprintf(sdsempty(),"-Reading from master: %s", @@ -1630,7 +1625,7 @@ char *sendSynchronousCommand(int flags, int fd, ...) { #define PSYNC_FULLRESYNC 3 #define PSYNC_NOT_SUPPORTED 4 #define PSYNC_TRY_LATER 5 -int slaveTryPartialResynchronization(int fd, int read_reply) { +int slaveTryPartialResynchronization(connection *conn, int read_reply) { char *psync_replid; char psync_offset[32]; sds reply; @@ -1655,18 +1650,18 @@ int slaveTryPartialResynchronization(int fd, int read_reply) { } /* Issue the PSYNC command */ - reply = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"PSYNC",psync_replid,psync_offset,NULL); + reply = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"PSYNC",psync_replid,psync_offset,NULL); if (reply != NULL) { serverLog(LL_WARNING,"Unable to send PSYNC to master: %s",reply); sdsfree(reply); - aeDeleteFileEvent(server.el,fd,AE_READABLE); + connSetReadHandler(conn, NULL); return PSYNC_WRITE_ERROR; } return PSYNC_WAIT_REPLY; } /* Reading half */ - reply = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + reply = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); if (sdslen(reply) == 0) { /* The master may send empty newlines after it receives PSYNC * and before to reply, just to keep the connection alive. */ @@ -1674,7 +1669,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) { return PSYNC_WAIT_REPLY; } - aeDeleteFileEvent(server.el,fd,AE_READABLE); + connSetReadHandler(conn, NULL); if (!strncmp(reply,"+FULLRESYNC",11)) { char *replid = NULL, *offset = NULL; @@ -1748,7 +1743,7 @@ int slaveTryPartialResynchronization(int fd, int read_reply) { /* Setup the replication to continue. */ sdsfree(reply); - replicationResurrectCachedMaster(fd); + replicationResurrectCachedMaster(conn); /* If this instance was restarted and we read the metadata to * PSYNC from the persistence file, our replication backlog could @@ -1790,29 +1785,23 @@ int slaveTryPartialResynchronization(int fd, int read_reply) { /* This handler fires when the non blocking connect was able to * establish a connection with the master. */ -void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { +void syncWithMaster(connection *conn) { char tmpfile[256], *err = NULL; int dfd = -1, maxtries = 5; - int sockerr = 0, psync_result; - socklen_t errlen = sizeof(sockerr); - UNUSED(el); - UNUSED(privdata); - UNUSED(mask); + int psync_result; /* If this event fired after the user turned the instance into a master * with SLAVEOF NO ONE we must just return ASAP. */ if (server.repl_state == REPL_STATE_NONE) { - close(fd); + connClose(conn); return; } /* Check for errors in the socket: after a non blocking connect() we * may find that the socket is in error state. */ - if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &sockerr, &errlen) == -1) - sockerr = errno; - if (sockerr) { + if (connGetState(conn) != CONN_STATE_CONNECTED) { serverLog(LL_WARNING,"Error condition on socket for SYNC: %s", - strerror(sockerr)); + connGetLastError(conn)); goto error; } @@ -1821,18 +1810,19 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { serverLog(LL_NOTICE,"Non blocking connect for SYNC fired the event."); /* Delete the writable event so that the readable event remains * registered and we can wait for the PONG reply. */ - aeDeleteFileEvent(server.el,fd,AE_WRITABLE); + connSetReadHandler(conn, syncWithMaster); + connSetWriteHandler(conn, NULL); server.repl_state = REPL_STATE_RECEIVE_PONG; /* Send the PING, don't check for errors at all, we have the timeout * that will take care about this. */ - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"PING",NULL); + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"PING",NULL); if (err) goto write_error; return; } /* Receive the PONG command. */ if (server.repl_state == REPL_STATE_RECEIVE_PONG) { - err = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); /* We accept only two replies as valid, a positive +PONG reply * (we just check for "+") or an authentication error. @@ -1857,13 +1847,13 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* AUTH with the master if required. */ if (server.repl_state == REPL_STATE_SEND_AUTH) { if (server.masteruser && server.masterauth) { - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"AUTH", + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"AUTH", server.masteruser,server.masterauth,NULL); if (err) goto write_error; server.repl_state = REPL_STATE_RECEIVE_AUTH; return; } else if (server.masterauth) { - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"AUTH",server.masterauth,NULL); + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"AUTH",server.masterauth,NULL); if (err) goto write_error; server.repl_state = REPL_STATE_RECEIVE_AUTH; return; @@ -1874,7 +1864,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Receive AUTH reply. */ if (server.repl_state == REPL_STATE_RECEIVE_AUTH) { - err = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); if (err[0] == '-') { serverLog(LL_WARNING,"Unable to AUTH to MASTER: %s",err); sdsfree(err); @@ -1889,7 +1879,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { if (server.repl_state == REPL_STATE_SEND_PORT) { sds port = sdsfromlonglong(server.slave_announce_port ? server.slave_announce_port : server.port); - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"REPLCONF", + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF", "listening-port",port, NULL); sdsfree(port); if (err) goto write_error; @@ -1900,7 +1890,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Receive REPLCONF listening-port reply. */ if (server.repl_state == REPL_STATE_RECEIVE_PORT) { - err = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); /* Ignore the error if any, not all the Redis versions support * REPLCONF listening-port. */ if (err[0] == '-') { @@ -1921,7 +1911,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Set the slave ip, so that Master's INFO command can list the * slave IP address port correctly in case of port forwarding or NAT. */ if (server.repl_state == REPL_STATE_SEND_IP) { - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"REPLCONF", + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF", "ip-address",server.slave_announce_ip, NULL); if (err) goto write_error; sdsfree(err); @@ -1931,7 +1921,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Receive REPLCONF ip-address reply. */ if (server.repl_state == REPL_STATE_RECEIVE_IP) { - err = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); /* Ignore the error if any, not all the Redis versions support * REPLCONF listening-port. */ if (err[0] == '-') { @@ -1949,7 +1939,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { * * The master will ignore capabilities it does not understand. */ if (server.repl_state == REPL_STATE_SEND_CAPA) { - err = sendSynchronousCommand(SYNC_CMD_WRITE,fd,"REPLCONF", + err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF", "capa","eof","capa","psync2",NULL); if (err) goto write_error; sdsfree(err); @@ -1959,7 +1949,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Receive CAPA reply. */ if (server.repl_state == REPL_STATE_RECEIVE_CAPA) { - err = sendSynchronousCommand(SYNC_CMD_READ,fd,NULL); + err = sendSynchronousCommand(SYNC_CMD_READ,conn,NULL); /* Ignore the error if any, not all the Redis versions support * REPLCONF capa. */ if (err[0] == '-') { @@ -1976,7 +1966,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { * and the global offset, to try a partial resync at the next * reconnection attempt. */ if (server.repl_state == REPL_STATE_SEND_PSYNC) { - if (slaveTryPartialResynchronization(fd,0) == PSYNC_WRITE_ERROR) { + if (slaveTryPartialResynchronization(conn,0) == PSYNC_WRITE_ERROR) { err = sdsnew("Write error sending the PSYNC command."); goto write_error; } @@ -1992,7 +1982,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { goto error; } - psync_result = slaveTryPartialResynchronization(fd,1); + psync_result = slaveTryPartialResynchronization(conn,1); if (psync_result == PSYNC_WAIT_REPLY) return; /* Try again later... */ /* If the master is in an transient error, we should try to PSYNC @@ -2021,7 +2011,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { * already populated. */ if (psync_result == PSYNC_NOT_SUPPORTED) { serverLog(LL_NOTICE,"Retrying with SYNC..."); - if (syncWrite(fd,"SYNC\r\n",6,server.repl_syncio_timeout*1000) == -1) { + if (connSyncWrite(conn,"SYNC\r\n",6,server.repl_syncio_timeout*1000) == -1) { serverLog(LL_WARNING,"I/O error writing to MASTER: %s", strerror(errno)); goto error; @@ -2046,12 +2036,13 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { } /* Setup the non blocking download of the bulk file. */ - if (aeCreateFileEvent(server.el,fd, AE_READABLE,readSyncBulkPayload,NULL) - == AE_ERR) + if (connSetReadHandler(conn, readSyncBulkPayload) + == C_ERR) { + char conninfo[CONN_INFO_LEN]; serverLog(LL_WARNING, - "Can't create readable event for SYNC: %s (fd=%d)", - strerror(errno),fd); + "Can't create readable event for SYNC: %s (%s)", + strerror(errno), connGetInfo(conn, conninfo, sizeof(conninfo))); goto error; } @@ -2063,16 +2054,15 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { return; error: - aeDeleteFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE); if (dfd != -1) close(dfd); - close(fd); + connClose(conn); + server.repl_transfer_s = NULL; if (server.repl_transfer_fd != -1) close(server.repl_transfer_fd); if (server.repl_transfer_tmpfile) zfree(server.repl_transfer_tmpfile); server.repl_transfer_tmpfile = NULL; server.repl_transfer_fd = -1; - server.repl_transfer_s = -1; server.repl_state = REPL_STATE_CONNECT; return; @@ -2083,26 +2073,18 @@ write_error: /* Handle sendSynchronousCommand(SYNC_CMD_WRITE) errors. */ } int connectWithMaster(void) { - int fd; - - fd = anetTcpNonBlockBestEffortBindConnect(NULL, - server.masterhost,server.masterport,NET_FIRST_BIND_ADDR); - if (fd == -1) { + server.repl_transfer_s = server.tls_replication ? connCreateTLS() : connCreateSocket(); + if (connConnect(server.repl_transfer_s, server.masterhost, server.masterport, + NET_FIRST_BIND_ADDR, syncWithMaster) == C_ERR) { serverLog(LL_WARNING,"Unable to connect to MASTER: %s", - strerror(errno)); + connGetLastError(server.repl_transfer_s)); + connClose(server.repl_transfer_s); + server.repl_transfer_s = NULL; return C_ERR; } - if (aeCreateFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE,syncWithMaster,NULL) == - AE_ERR) - { - close(fd); - serverLog(LL_WARNING,"Can't create readable event for SYNC"); - return C_ERR; - } server.repl_transfer_lastio = server.unixtime; - server.repl_transfer_s = fd; server.repl_state = REPL_STATE_CONNECTING; return C_OK; } @@ -2112,11 +2094,8 @@ int connectWithMaster(void) { * Never call this function directly, use cancelReplicationHandshake() instead. */ void undoConnectWithMaster(void) { - int fd = server.repl_transfer_s; - - aeDeleteFileEvent(server.el,fd,AE_READABLE|AE_WRITABLE); - close(fd); - server.repl_transfer_s = -1; + connClose(server.repl_transfer_s); + server.repl_transfer_s = NULL; } /* Abort the async download of the bulk dataset while SYNC-ing with master. @@ -2301,7 +2280,7 @@ void roleCommand(client *c) { char ip[NET_IP_STR_LEN], *slaveip = slave->slave_ip; if (slaveip[0] == '\0') { - if (anetPeerToString(slave->fd,ip,sizeof(ip),NULL) == -1) + if (connPeerToString(slave->conn,ip,sizeof(ip),NULL) == -1) continue; slaveip = ip; } @@ -2423,7 +2402,7 @@ void replicationCacheMasterUsingMyself(void) { /* The master client we create can be set to any DBID, because * the new master will start its replication stream with SELECT. */ server.master_initial_offset = server.master_repl_offset; - replicationCreateMasterClient(-1,-1); + replicationCreateMasterClient(NULL,-1); /* Use our own ID / offset. */ memcpy(server.master->replid, server.replid, sizeof(server.replid)); @@ -2452,10 +2431,11 @@ void replicationDiscardCachedMaster(void) { * This function is called when successfully setup a partial resynchronization * so the stream of data that we'll receive will start from were this * master left. */ -void replicationResurrectCachedMaster(int newfd) { +void replicationResurrectCachedMaster(connection *conn) { server.master = server.cached_master; server.cached_master = NULL; - server.master->fd = newfd; + server.master->conn = conn; + connSetPrivateData(server.master->conn, server.master); server.master->flags &= ~(CLIENT_CLOSE_AFTER_REPLY|CLIENT_CLOSE_ASAP); server.master->authenticated = 1; server.master->lastinteraction = server.unixtime; @@ -2464,8 +2444,7 @@ void replicationResurrectCachedMaster(int newfd) { /* Re-add to the list of clients. */ linkClient(server.master); - if (aeCreateFileEvent(server.el, newfd, AE_READABLE, - readQueryFromClient, server.master)) { + if (connSetReadHandler(server.master->conn, readQueryFromClient)) { serverLog(LL_WARNING,"Error resurrecting the cached master, impossible to add the readable handler: %s", strerror(errno)); freeClientAsync(server.master); /* Close ASAP. */ } @@ -2473,8 +2452,7 @@ void replicationResurrectCachedMaster(int newfd) { /* We may also need to install the write handler as well if there is * pending data in the write buffers. */ if (clientHasPendingReplies(server.master)) { - if (aeCreateFileEvent(server.el, newfd, AE_WRITABLE, - sendReplyToClient, server.master)) { + if (connSetWriteHandler(server.master->conn, sendReplyToClient)) { serverLog(LL_WARNING,"Error resurrecting the cached master, impossible to add the writable handler: %s", strerror(errno)); freeClientAsync(server.master); /* Close ASAP. */ } @@ -2844,9 +2822,7 @@ void replicationCron(void) { server.rdb_child_type != RDB_CHILD_TYPE_SOCKET)); if (is_presync) { - if (write(slave->fd, "\n", 1) == -1) { - /* Don't worry about socket errors, it's just a ping. */ - } + connWrite(slave->conn, "\n", 1); } } diff --git a/src/rio.c b/src/rio.c index bdbc5d0e9..5e3b4a06e 100644 --- a/src/rio.c +++ b/src/rio.c @@ -159,13 +159,13 @@ void rioInitWithFile(rio *r, FILE *fp) { r->io.file.autosync = 0; } -/* ------------------- File descriptor implementation ------------------- +/* ------------------- Connection implementation ------------------- * We use this RIO implemetnation when reading an RDB file directly from - * the socket to the memory via rdbLoadRio(), thus this implementation - * only implements reading from a file descriptor that is, normally, + * the connection to the memory via rdbLoadRio(), thus this implementation + * only implements reading from a connection that is, normally, * just a socket. */ -static size_t rioFdWrite(rio *r, const void *buf, size_t len) { +static size_t rioConnWrite(rio *r, const void *buf, size_t len) { UNUSED(r); UNUSED(buf); UNUSED(len); @@ -173,72 +173,72 @@ static size_t rioFdWrite(rio *r, const void *buf, size_t len) { } /* Returns 1 or 0 for success/failure. */ -static size_t rioFdRead(rio *r, void *buf, size_t len) { - size_t avail = sdslen(r->io.fd.buf)-r->io.fd.pos; +static size_t rioConnRead(rio *r, void *buf, size_t len) { + size_t avail = sdslen(r->io.conn.buf)-r->io.conn.pos; /* If the buffer is too small for the entire request: realloc. */ - if (sdslen(r->io.fd.buf) + sdsavail(r->io.fd.buf) < len) - r->io.fd.buf = sdsMakeRoomFor(r->io.fd.buf, len - sdslen(r->io.fd.buf)); + if (sdslen(r->io.conn.buf) + sdsavail(r->io.conn.buf) < len) + r->io.conn.buf = sdsMakeRoomFor(r->io.conn.buf, len - sdslen(r->io.conn.buf)); /* If the remaining unused buffer is not large enough: memmove so that we * can read the rest. */ - if (len > avail && sdsavail(r->io.fd.buf) < len - avail) { - sdsrange(r->io.fd.buf, r->io.fd.pos, -1); - r->io.fd.pos = 0; + if (len > avail && sdsavail(r->io.conn.buf) < len - avail) { + sdsrange(r->io.conn.buf, r->io.conn.pos, -1); + r->io.conn.pos = 0; } /* If we don't already have all the data in the sds, read more */ - while (len > sdslen(r->io.fd.buf) - r->io.fd.pos) { - size_t buffered = sdslen(r->io.fd.buf) - r->io.fd.pos; + while (len > sdslen(r->io.conn.buf) - r->io.conn.pos) { + size_t buffered = sdslen(r->io.conn.buf) - r->io.conn.pos; size_t toread = len - buffered; /* Read either what's missing, or PROTO_IOBUF_LEN, the bigger of * the two. */ if (toread < PROTO_IOBUF_LEN) toread = PROTO_IOBUF_LEN; - if (toread > sdsavail(r->io.fd.buf)) toread = sdsavail(r->io.fd.buf); - if (r->io.fd.read_limit != 0 && - r->io.fd.read_so_far + buffered + toread > r->io.fd.read_limit) + if (toread > sdsavail(r->io.conn.buf)) toread = sdsavail(r->io.conn.buf); + if (r->io.conn.read_limit != 0 && + r->io.conn.read_so_far + buffered + toread > r->io.conn.read_limit) { - if (r->io.fd.read_limit >= r->io.fd.read_so_far - buffered) - toread = r->io.fd.read_limit - r->io.fd.read_so_far - buffered; + if (r->io.conn.read_limit >= r->io.conn.read_so_far - buffered) + toread = r->io.conn.read_limit - r->io.conn.read_so_far - buffered; else { errno = EOVERFLOW; return 0; } } - int retval = read(r->io.fd.fd, - (char*)r->io.fd.buf + sdslen(r->io.fd.buf), + int retval = connRead(r->io.conn.conn, + (char*)r->io.conn.buf + sdslen(r->io.conn.buf), toread); if (retval <= 0) { if (errno == EWOULDBLOCK) errno = ETIMEDOUT; return 0; } - sdsIncrLen(r->io.fd.buf, retval); + sdsIncrLen(r->io.conn.buf, retval); } - memcpy(buf, (char*)r->io.fd.buf + r->io.fd.pos, len); - r->io.fd.read_so_far += len; - r->io.fd.pos += len; + memcpy(buf, (char*)r->io.conn.buf + r->io.conn.pos, len); + r->io.conn.read_so_far += len; + r->io.conn.pos += len; return len; } /* Returns read/write position in file. */ -static off_t rioFdTell(rio *r) { - return r->io.fd.read_so_far; +static off_t rioConnTell(rio *r) { + return r->io.conn.read_so_far; } /* Flushes any buffer to target device if applicable. Returns 1 on success * and 0 on failures. */ -static int rioFdFlush(rio *r) { +static int rioConnFlush(rio *r) { /* Our flush is implemented by the write method, that recognizes a * buffer set to NULL with a count of zero as a flush request. */ - return rioFdWrite(r,NULL,0); + return rioConnWrite(r,NULL,0); } -static const rio rioFdIO = { - rioFdRead, - rioFdWrite, - rioFdTell, - rioFdFlush, +static const rio rioConnIO = { + rioConnRead, + rioConnWrite, + rioConnTell, + rioConnFlush, NULL, /* update_checksum */ 0, /* current checksum */ 0, /* flags */ @@ -249,27 +249,27 @@ static const rio rioFdIO = { /* Create an RIO that implements a buffered read from an fd * read_limit argument stops buffering when the reaching the limit. */ -void rioInitWithFd(rio *r, int fd, size_t read_limit) { - *r = rioFdIO; - r->io.fd.fd = fd; - r->io.fd.pos = 0; - r->io.fd.read_limit = read_limit; - r->io.fd.read_so_far = 0; - r->io.fd.buf = sdsnewlen(NULL, PROTO_IOBUF_LEN); - sdsclear(r->io.fd.buf); +void rioInitWithConn(rio *r, connection *conn, size_t read_limit) { + *r = rioConnIO; + r->io.conn.conn = conn; + r->io.conn.pos = 0; + r->io.conn.read_limit = read_limit; + r->io.conn.read_so_far = 0; + r->io.conn.buf = sdsnewlen(NULL, PROTO_IOBUF_LEN); + sdsclear(r->io.conn.buf); } /* Release the RIO tream. Optionally returns the unread buffered data * when the SDS pointer 'remaining' is passed. */ -void rioFreeFd(rio *r, sds *remaining) { - if (remaining && (size_t)r->io.fd.pos < sdslen(r->io.fd.buf)) { - if (r->io.fd.pos > 0) sdsrange(r->io.fd.buf, r->io.fd.pos, -1); - *remaining = r->io.fd.buf; +void rioFreeConn(rio *r, sds *remaining) { + if (remaining && (size_t)r->io.conn.pos < sdslen(r->io.conn.buf)) { + if (r->io.conn.pos > 0) sdsrange(r->io.conn.buf, r->io.conn.pos, -1); + *remaining = r->io.conn.buf; } else { - sdsfree(r->io.fd.buf); + sdsfree(r->io.conn.buf); if (remaining) *remaining = NULL; } - r->io.fd.buf = NULL; + r->io.conn.buf = NULL; } /* ------------------- File descriptors set implementation ------------------ @@ -294,14 +294,14 @@ static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) { /* To start we always append to our buffer. If it gets larger than * a given size, we actually write to the sockets. */ if (len) { - r->io.fdset.buf = sdscatlen(r->io.fdset.buf,buf,len); + r->io.connset.buf = sdscatlen(r->io.connset.buf,buf,len); len = 0; /* Prevent entering the while below if we don't flush. */ - if (sdslen(r->io.fdset.buf) > PROTO_IOBUF_LEN) doflush = 1; + if (sdslen(r->io.connset.buf) > PROTO_IOBUF_LEN) doflush = 1; } if (doflush) { - p = (unsigned char*) r->io.fdset.buf; - len = sdslen(r->io.fdset.buf); + p = (unsigned char*) r->io.connset.buf; + len = sdslen(r->io.connset.buf); } /* Write in little chunchs so that when there are big writes we @@ -310,8 +310,8 @@ static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) { while(len) { size_t count = len < 1024 ? len : 1024; int broken = 0; - for (j = 0; j < r->io.fdset.numfds; j++) { - if (r->io.fdset.state[j] != 0) { + for (j = 0; j < r->io.connset.numconns; j++) { + if (r->io.connset.state[j] != 0) { /* Skip FDs alraedy in error. */ broken++; continue; @@ -321,7 +321,7 @@ static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) { * of short writes. */ size_t nwritten = 0; while(nwritten != count) { - retval = write(r->io.fdset.fds[j],p+nwritten,count-nwritten); + retval = connWrite(r->io.connset.conns[j],p+nwritten,count-nwritten); if (retval <= 0) { /* With blocking sockets, which is the sole user of this * rio target, EWOULDBLOCK is returned only because of @@ -335,17 +335,17 @@ static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) { if (nwritten != count) { /* Mark this FD as broken. */ - r->io.fdset.state[j] = errno; - if (r->io.fdset.state[j] == 0) r->io.fdset.state[j] = EIO; + r->io.connset.state[j] = errno; + if (r->io.connset.state[j] == 0) r->io.connset.state[j] = EIO; } } - if (broken == r->io.fdset.numfds) return 0; /* All the FDs in error. */ + if (broken == r->io.connset.numconns) return 0; /* All the FDs in error. */ p += count; len -= count; - r->io.fdset.pos += count; + r->io.connset.pos += count; } - if (doflush) sdsclear(r->io.fdset.buf); + if (doflush) sdsclear(r->io.connset.buf); return 1; } @@ -359,7 +359,7 @@ static size_t rioFdsetRead(rio *r, void *buf, size_t len) { /* Returns read/write position in file. */ static off_t rioFdsetTell(rio *r) { - return r->io.fdset.pos; + return r->io.connset.pos; } /* Flushes any buffer to target device if applicable. Returns 1 on success @@ -383,24 +383,24 @@ static const rio rioFdsetIO = { { { NULL, 0 } } /* union for io-specific vars */ }; -void rioInitWithFdset(rio *r, int *fds, int numfds) { +void rioInitWithConnset(rio *r, connection **conns, int numconns) { int j; *r = rioFdsetIO; - r->io.fdset.fds = zmalloc(sizeof(int)*numfds); - r->io.fdset.state = zmalloc(sizeof(int)*numfds); - memcpy(r->io.fdset.fds,fds,sizeof(int)*numfds); - for (j = 0; j < numfds; j++) r->io.fdset.state[j] = 0; - r->io.fdset.numfds = numfds; - r->io.fdset.pos = 0; - r->io.fdset.buf = sdsempty(); + r->io.connset.conns = zmalloc(sizeof(connection *)*numconns); + r->io.connset.state = zmalloc(sizeof(int)*numconns); + memcpy(r->io.connset.conns,conns,sizeof(connection *)*numconns); + for (j = 0; j < numconns; j++) r->io.connset.state[j] = 0; + r->io.connset.numconns = numconns; + r->io.connset.pos = 0; + r->io.connset.buf = sdsempty(); } /* release the rio stream. */ -void rioFreeFdset(rio *r) { - zfree(r->io.fdset.fds); - zfree(r->io.fdset.state); - sdsfree(r->io.fdset.buf); +void rioFreeConnset(rio *r) { + zfree(r->io.connset.conns); + zfree(r->io.connset.state); + sdsfree(r->io.connset.buf); } /* ---------------------------- Generic functions ---------------------------- */ diff --git a/src/rio.h b/src/rio.h index eb7a05748..fdde7c20e 100644 --- a/src/rio.h +++ b/src/rio.h @@ -35,6 +35,7 @@ #include #include #include "sds.h" +#include "connection.h" #define RIO_FLAG_READ_ERROR (1<<0) #define RIO_FLAG_WRITE_ERROR (1<<1) @@ -76,22 +77,22 @@ struct _rio { off_t buffered; /* Bytes written since last fsync. */ off_t autosync; /* fsync after 'autosync' bytes written. */ } file; - /* file descriptor */ + /* Connection object */ struct { - int fd; /* File descriptor. */ + connection *conn; /* Connection */ off_t pos; /* pos in buf that was returned */ sds buf; /* buffered data */ size_t read_limit; /* don't allow to buffer/read more than that */ size_t read_so_far; /* amount of data read from the rio (not buffered) */ - } fd; + } conn; /* Multiple FDs target (used to write to N sockets). */ struct { - int *fds; /* File descriptors. */ - int *state; /* Error state of each fd. 0 (if ok) or errno. */ - int numfds; + connection **conns; /* Connections */ + int *state; /* Error state of each fd. 0 (if ok) or errno. */ + int numconns; off_t pos; sds buf; - } fdset; + } connset; } io; }; @@ -159,11 +160,11 @@ static inline void rioClearErrors(rio *r) { void rioInitWithFile(rio *r, FILE *fp); void rioInitWithBuffer(rio *r, sds s); -void rioInitWithFd(rio *r, int fd, size_t read_limit); -void rioInitWithFdset(rio *r, int *fds, int numfds); +void rioInitWithConn(rio *r, connection *conn, size_t read_limit); +void rioInitWithConnset(rio *r, connection **conns, int numconns); -void rioFreeFdset(rio *r); -void rioFreeFd(rio *r, sds* out_remainingBufferedData); +void rioFreeConnset(rio *r); +void rioFreeConn(rio *r, sds* out_remainingBufferedData); size_t rioWriteBulkCount(rio *r, char prefix, long count); size_t rioWriteBulkString(rio *r, const char *buf, size_t len); diff --git a/src/scripting.c b/src/scripting.c index 032bfdf10..6c1b89dbe 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -58,7 +58,7 @@ sds ldbCatStackValue(sds s, lua_State *lua, int idx); #define LDB_BREAKPOINTS_MAX 64 /* Max number of breakpoints. */ #define LDB_MAX_LEN_DEFAULT 256 /* Default len limit for replies / var dumps. */ struct ldbState { - int fd; /* Socket of the debugging client. */ + connection *conn; /* Connection of the debugging client. */ int active; /* Are we debugging EVAL right now? */ int forked; /* Is this a fork()ed debugging session? */ list *logs; /* List of messages to send to the client. */ @@ -1109,7 +1109,7 @@ void scriptingInit(int setup) { * Note: there is no need to create it again when this function is called * by scriptingReset(). */ if (server.lua_client == NULL) { - server.lua_client = createClient(-1); + server.lua_client = createClient(NULL); server.lua_client->flags |= CLIENT_LUA; } @@ -1593,7 +1593,7 @@ NULL /* Initialize Lua debugger data structures. */ void ldbInit(void) { - ldb.fd = -1; + ldb.conn = NULL; ldb.active = 0; ldb.logs = listCreate(); listSetFreeMethod(ldb.logs,(void (*)(void*))sdsfree); @@ -1615,7 +1615,7 @@ void ldbFlushLog(list *log) { void ldbEnable(client *c) { c->flags |= CLIENT_LUA_DEBUG; ldbFlushLog(ldb.logs); - ldb.fd = c->fd; + ldb.conn = c->conn; ldb.step = 1; ldb.bpcount = 0; ldb.luabp = 0; @@ -1670,7 +1670,7 @@ void ldbSendLogs(void) { proto = sdscatlen(proto,"\r\n",2); listDelNode(ldb.logs,ln); } - if (write(ldb.fd,proto,sdslen(proto)) == -1) { + if (connWrite(ldb.conn,proto,sdslen(proto)) == -1) { /* Avoid warning. We don't check the return value of write() * since the next read() will catch the I/O error and will * close the debugging session. */ @@ -1723,8 +1723,8 @@ int ldbStartSession(client *c) { } /* Setup our debugging session. */ - anetBlock(NULL,ldb.fd); - anetSendTimeout(NULL,ldb.fd,5000); + connBlock(ldb.conn); + connSendTimeout(ldb.conn,5000); ldb.active = 1; /* First argument of EVAL is the script itself. We split it into different @@ -1751,7 +1751,7 @@ void ldbEndSession(client *c) { /* If it's a fork()ed session, we just exit. */ if (ldb.forked) { - writeToClient(c->fd, c, 0); + writeToClient(c,0); serverLog(LL_WARNING,"Lua debugging session child exiting"); exitFromChild(0); } else { @@ -1760,8 +1760,8 @@ void ldbEndSession(client *c) { } /* Otherwise let's restore client's state. */ - anetNonBlock(NULL,ldb.fd); - anetSendTimeout(NULL,ldb.fd,0); + connNonBlock(ldb.conn); + connSendTimeout(ldb.conn,0); /* Close the client connectin after sending the final EVAL reply * in order to signal the end of the debugging session. */ @@ -2332,7 +2332,7 @@ int ldbRepl(lua_State *lua) { while(1) { while((argv = ldbReplParseCommand(&argc)) == NULL) { char buf[1024]; - int nread = read(ldb.fd,buf,sizeof(buf)); + int nread = connRead(ldb.conn,buf,sizeof(buf)); if (nread <= 0) { /* Make sure the script runs without user input since the * client is no longer connected. */ diff --git a/src/sentinel.c b/src/sentinel.c index 92ea75436..95f189485 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -30,6 +30,10 @@ #include "server.h" #include "hiredis.h" +#ifdef USE_OPENSSL +#include "openssl/ssl.h" +#include "hiredis_ssl.h" +#endif #include "async.h" #include @@ -40,6 +44,10 @@ extern char **environ; +#ifdef USE_OPENSSL +extern SSL_CTX *redis_tls_ctx; +#endif + #define REDIS_SENTINEL_PORT 26379 /* ======================== Sentinel global state =========================== */ @@ -1995,6 +2003,19 @@ void sentinelSetClientName(sentinelRedisInstance *ri, redisAsyncContext *c, char } } +static int instanceLinkNegotiateTLS(redisAsyncContext *context) { +#ifndef USE_OPENSSL + (void) link; +#else + if (!redis_tls_ctx) return C_ERR; + SSL *ssl = SSL_new(redis_tls_ctx); + if (!ssl) return C_ERR; + + if (redisInitiateSSL(&context->c, ssl) == REDIS_ERR) return C_ERR; +#endif + return C_OK; +} + /* Create the async connections for the instance link if the link * is disconnected. Note that link->disconnected is true even if just * one of the two links (commands and pub/sub) is missing. */ @@ -2010,7 +2031,11 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { /* Commands connection. */ if (link->cc == NULL) { link->cc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR); - if (link->cc->err) { + if (!link->cc->err && server.tls_replication && + (instanceLinkNegotiateTLS(link->cc) == C_ERR)) { + sentinelEvent(LL_DEBUG,"-cmd-link-reconnection",ri,"%@ #Failed to initialize TLS"); + instanceLinkCloseConnection(link,link->cc); + } else if (link->cc->err) { sentinelEvent(LL_DEBUG,"-cmd-link-reconnection",ri,"%@ #%s", link->cc->errstr); instanceLinkCloseConnection(link,link->cc); @@ -2033,7 +2058,10 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { /* Pub / Sub */ if ((ri->flags & (SRI_MASTER|SRI_SLAVE)) && link->pc == NULL) { link->pc = redisAsyncConnectBind(ri->addr->ip,ri->addr->port,NET_FIRST_BIND_ADDR); - if (link->pc->err) { + if (!link->pc->err && server.tls_replication && + (instanceLinkNegotiateTLS(link->pc) == C_ERR)) { + sentinelEvent(LL_DEBUG,"-pubsub-link-reconnection",ri,"%@ #Failed to initialize TLS"); + } else if (link->pc->err) { sentinelEvent(LL_DEBUG,"-pubsub-link-reconnection",ri,"%@ #%s", link->pc->errstr); instanceLinkCloseConnection(link,link->pc); diff --git a/src/server.c b/src/server.c index fb308e5f8..930a01723 100644 --- a/src/server.c +++ b/src/server.c @@ -2229,11 +2229,13 @@ void initServerConfig(void) { server.dynamic_hz = CONFIG_DEFAULT_DYNAMIC_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = CONFIG_DEFAULT_SERVER_PORT; + server.tls_port = CONFIG_DEFAULT_SERVER_TLS_PORT; server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; server.bindaddr_count = 0; server.unixsocket = NULL; server.unixsocketperm = CONFIG_DEFAULT_UNIX_SOCKET_PERM; server.ipfd_count = 0; + server.tlsfd_count = 0; server.sofd = -1; server.protected_mode = CONFIG_DEFAULT_PROTECTED_MODE; server.gopher_enabled = CONFIG_DEFAULT_GOPHER_ENABLED; @@ -2349,7 +2351,7 @@ void initServerConfig(void) { server.repl_state = REPL_STATE_NONE; server.repl_transfer_tmpfile = NULL; server.repl_transfer_fd = -1; - server.repl_transfer_s = -1; + server.repl_transfer_s = NULL; server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA; server.repl_slave_ro = CONFIG_DEFAULT_SLAVE_READ_ONLY; @@ -2430,6 +2432,9 @@ void initServerConfig(void) { * script to the slave / AOF. This is the new way starting from * Redis 5. However it is possible to revert it via redis.conf. */ server.lua_always_replicate_commands = 1; + + /* TLS */ + server.tls_auth_clients = 1; } extern char **environ; @@ -2746,6 +2751,11 @@ void initServer(void) { server.clients_paused = 0; server.system_memory_size = zmalloc_get_memory_size(); + if (server.tls_port && tlsConfigureServer() == C_ERR) { + serverLog(LL_WARNING, "Failed to configure TLS. Check logs for more info."); + exit(1); + } + createSharedObjects(); adjustOpenFilesLimit(); server.el = aeCreateEventLoop(server.maxclients+CONFIG_FDSET_INCR); @@ -2761,6 +2771,9 @@ void initServer(void) { if (server.port != 0 && listenToPort(server.port,server.ipfd,&server.ipfd_count) == C_ERR) exit(1); + if (server.tls_port != 0 && + listenToPort(server.tls_port,server.tlsfd,&server.tlsfd_count) == C_ERR) + exit(1); /* Open the listening Unix domain socket. */ if (server.unixsocket != NULL) { @@ -2775,7 +2788,7 @@ void initServer(void) { } /* Abort if there are no listening sockets at all. */ - if (server.ipfd_count == 0 && server.sofd < 0) { + if (server.ipfd_count == 0 && server.tlsfd_count == 0 && server.sofd < 0) { serverLog(LL_WARNING, "Configured to not listen anywhere, exiting."); exit(1); } @@ -2845,6 +2858,14 @@ void initServer(void) { "Unrecoverable error creating server.ipfd file event."); } } + for (j = 0; j < server.tlsfd_count; j++) { + if (aeCreateFileEvent(server.el, server.tlsfd[j], AE_READABLE, + acceptTLSHandler,NULL) == AE_ERR) + { + serverPanic( + "Unrecoverable error creating server.tlsfd file event."); + } + } if (server.sofd > 0 && aeCreateFileEvent(server.el,server.sofd,AE_READABLE, acceptUnixHandler,NULL) == AE_ERR) serverPanic("Unrecoverable error creating server.sofd file event."); @@ -3536,6 +3557,7 @@ void closeListeningSockets(int unlink_unix_socket) { int j; for (j = 0; j < server.ipfd_count; j++) close(server.ipfd[j]); + for (j = 0; j < server.tlsfd_count; j++) close(server.tlsfd[j]); if (server.sofd != -1) close(server.sofd); if (server.cluster_enabled) for (j = 0; j < server.cfd_count; j++) close(server.cfd[j]); @@ -4274,7 +4296,7 @@ sds genRedisInfoString(char *section) { long lag = 0; if (slaveip[0] == '\0') { - if (anetPeerToString(slave->fd,ip,sizeof(ip),&port) == -1) + if (connPeerToString(slave->conn,ip,sizeof(ip),&port) == -1) continue; slaveip = ip; } @@ -4516,7 +4538,7 @@ void redisAsciiArt(void) { redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, (sizeof(long) == 8) ? "64" : "32", - mode, server.port, + mode, server.port ? server.port : server.tls_port, (long) getpid() ); serverLogRaw(LL_NOTICE|LL_RAW,buf); @@ -4642,7 +4664,7 @@ void redisSetProcTitle(char *title) { setproctitle("%s %s:%d%s", title, server.bindaddr_count ? server.bindaddr[0] : "*", - server.port, + server.port ? server.port : server.tls_port, server_mode); #else UNUSED(title); @@ -4793,6 +4815,7 @@ int main(int argc, char **argv) { ACLInit(); /* The ACL subsystem must be initialized ASAP because the basic networking code and client creation depends on it. */ moduleInitModulesSystem(); + tlsInit(); /* Store the executable path and arguments in a safe place in order * to be able to restart the server later. */ @@ -4925,7 +4948,7 @@ int main(int argc, char **argv) { exit(1); } } - if (server.ipfd_count > 0) + if (server.ipfd_count > 0 || server.tlsfd_count > 0) serverLog(LL_NOTICE,"Ready to accept connections"); if (server.sofd > 0) serverLog(LL_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); diff --git a/src/server.h b/src/server.h index 31d253066..1b41c9ac5 100644 --- a/src/server.h +++ b/src/server.h @@ -66,6 +66,7 @@ typedef long long mstime_t; /* millisecond time type. */ #include "quicklist.h" /* Lists are encoded as linked lists of N-elements flat arrays */ #include "rax.h" /* Radix tree */ +#include "connection.h" /* Connection abstraction */ /* Following includes allow test functions to be called from Redis main() */ #include "zipmap.h" @@ -84,6 +85,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_MAX_HZ 500 #define MAX_CLIENTS_PER_CLOCK_TICK 200 /* HZ is adapted based on that. */ #define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port. */ +#define CONFIG_DEFAULT_SERVER_TLS_PORT 0 /* TCP port. */ #define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog. */ #define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* Default client timeout: infinite */ #define CONFIG_DEFAULT_DBNUM 16 @@ -818,7 +820,7 @@ typedef struct user { * Clients are taken in a linked list. */ typedef struct client { uint64_t id; /* Client incremental unique ID. */ - int fd; /* Client socket. */ + connection *conn; int resp; /* RESP protocol version. Can be 2 or 3. */ redisDb *db; /* Pointer to currently SELECTed DB. */ robj *name; /* As set by CLIENT SETNAME. */ @@ -1078,6 +1080,7 @@ struct redisServer { to be processed. */ /* Networking */ int port; /* TCP listening port */ + int tls_port; /* TLS listening port */ int tcp_backlog; /* TCP listen() backlog */ char *bindaddr[CONFIG_BINDADDR_MAX]; /* Addresses we should bind to */ int bindaddr_count; /* Number of addresses in server.bindaddr[] */ @@ -1085,6 +1088,8 @@ struct redisServer { mode_t unixsocketperm; /* UNIX socket permission */ int ipfd[CONFIG_BINDADDR_MAX]; /* TCP socket file descriptors */ int ipfd_count; /* Used slots in ipfd[] */ + int tlsfd[CONFIG_BINDADDR_MAX]; /* TLS socket file descriptors */ + int tlsfd_count; /* Used slots in tlsfd[] */ int sofd; /* Unix socket file descriptor */ int cfd[CONFIG_BINDADDR_MAX];/* Cluster bus listening socket */ int cfd_count; /* Used slots in cfd[] */ @@ -1287,7 +1292,7 @@ struct redisServer { off_t repl_transfer_size; /* Size of RDB to read from master during sync. */ off_t repl_transfer_read; /* Amount of RDB read from master during sync. */ off_t repl_transfer_last_fsync_off; /* Offset when we fsync-ed last time. */ - int repl_transfer_s; /* Slave -> Master SYNC socket */ + connection *repl_transfer_s; /* Slave -> Master SYNC connection */ int repl_transfer_fd; /* Slave -> Master SYNC temp file descriptor */ char *repl_transfer_tmpfile; /* Slave-> master SYNC temp file name */ time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */ @@ -1410,6 +1415,14 @@ struct redisServer { int watchdog_period; /* Software watchdog period in ms. 0 = off */ /* System hardware info */ size_t system_memory_size; /* Total memory in system as reported by OS */ + /* TLS Configuration */ + int tls_cluster; + int tls_replication; + char *tls_cert_file; + char *tls_key_file; + char *tls_dh_params_file; + char *tls_ca_cert_file; + int tls_auth_clients; }; typedef struct pubsubPattern { @@ -1553,12 +1566,12 @@ size_t redisPopcount(void *s, long count); void redisSetProcTitle(char *title); /* networking.c -- Networking and Client related operations */ -client *createClient(int fd); +client *createClient(connection *conn); void closeTimedoutClients(void); void freeClient(client *c); void freeClientAsync(client *c); void resetClient(client *c); -void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask); +void sendReplyToClient(connection *conn); void *addReplyDeferredLen(client *c); void setDeferredArrayLen(client *c, void *node, long length); void setDeferredMapLen(client *c, void *node, long length); @@ -1570,8 +1583,9 @@ void processInputBufferAndReplicate(client *c); void processGopherRequest(client *c); void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask); +void acceptTLSHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask); +void readQueryFromClient(connection *conn); void addReplyNull(client *c); void addReplyNullArray(client *c); void addReplyBool(client *c, int b); @@ -1629,7 +1643,7 @@ int handleClientsWithPendingReadsUsingThreads(void); int stopThreadedIOIfNeeded(void); int clientHasPendingReplies(client *c); void unlinkClient(client *c); -int writeToClient(int fd, client *c, int handler_installed); +int writeToClient(client *c, int handler_installed); void linkClient(client *c); void protectClient(client *c); void unprotectClient(client *c); @@ -2343,6 +2357,12 @@ void mixDigest(unsigned char *digest, void *ptr, size_t len); void xorDigest(unsigned char *digest, void *ptr, size_t len); int populateCommandTableParseFlags(struct redisCommand *c, char *strflags); +/* TLS stuff */ +void tlsInit(void); +int tlsConfigureServer(void); +int tlsConfigure(const char *cert_file, const char *key_file, + const char *dh_params_file, const char *ca_cert_file); + #define redisDebug(fmt, ...) \ printf("DEBUG %s:%d > " fmt "\n", __FILE__, __LINE__, __VA_ARGS__) #define redisDebugMark() \ diff --git a/src/tls.c b/src/tls.c new file mode 100644 index 000000000..dabb2ee0f --- /dev/null +++ b/src/tls.c @@ -0,0 +1,669 @@ +/* + * Copyright (c) 2019, Redis Labs + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +#include "server.h" +#include "connhelpers.h" + +#ifdef USE_OPENSSL + +#include +#include +#include + +extern ConnectionType CT_Socket; + +SSL_CTX *redis_tls_ctx; + +void tlsInit(void) { + ERR_load_crypto_strings(); + SSL_load_error_strings(); + SSL_library_init(); + + if (!RAND_poll()) { + serverLog(LL_WARNING, "OpenSSL: Failed to seed random number generator."); + } +} + +int tlsConfigureServer(void) { + return tlsConfigure(server.tls_cert_file, server.tls_key_file, + server.tls_dh_params_file, server.tls_ca_cert_file); +} + +/* Attempt to configure/reconfigure TLS. This operation is atomic and will + * leave the SSL_CTX unchanged if fails. + */ +int tlsConfigure(const char *cert_file, const char *key_file, + const char *dh_params_file, const char *ca_cert_file) { + + char errbuf[256]; + SSL_CTX *ctx = NULL; + + if (!cert_file) { + serverLog(LL_WARNING, "No tls-cert-file configured!"); + goto error; + } + + if (!key_file) { + serverLog(LL_WARNING, "No tls-key-file configured!"); + goto error; + } + + if (!ca_cert_file) { + serverLog(LL_WARNING, "No tls-ca-cert-file configured!"); + goto error; + } + + ctx = SSL_CTX_new(TLS_method()); + SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); + SSL_CTX_set_ecdh_auto(ctx, 1); + + if (SSL_CTX_use_certificate_file(ctx, cert_file, SSL_FILETYPE_PEM) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load certificate: %s: %s", cert_file, errbuf); + goto error; + } + + if (SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load private key: %s: %s", key_file, errbuf); + goto error; + } + + if (SSL_CTX_load_verify_locations(ctx, ca_cert_file, NULL) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load CA certificate(s) file: %s: %s", ca_cert_file, errbuf); + goto error; + } + + if (dh_params_file) { + FILE *dhfile = fopen(dh_params_file, "r"); + DH *dh = NULL; + if (!dhfile) { + serverLog(LL_WARNING, "Failed to load %s: %s", dh_params_file, strerror(errno)); + goto error; + } + + dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); + fclose(dhfile); + if (!dh) { + serverLog(LL_WARNING, "%s: failed to read DH params.", dh_params_file); + goto error; + } + + if (SSL_CTX_set_tmp_dh(ctx, dh) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", dh_params_file, errbuf); + DH_free(dh); + goto error; + } + + DH_free(dh); + } + + if (ctx_config->ciphers && !SSL_CTX_set_cipher_list(ctx, ctx_config->ciphers)) { + serverLog(LL_WARNING, "Failed to configure ciphers: %s", ctx_config->ciphers); + goto error; + } + +#ifdef TLS1_3_VERSION + if (ctx_config->ciphersuites && !SSL_CTX_set_ciphersuites(ctx, ctx_config->ciphersuites)) { + serverLog(LL_WARNING, "Failed to configure ciphersuites: %s", ctx_config->ciphersuites); + goto error; + } +#endif + + SSL_CTX_free(redis_tls_ctx); + redis_tls_ctx = ctx; + + return C_OK; + +error: + if (ctx) SSL_CTX_free(ctx); + return C_ERR; +} + +#ifdef TLS_DEBUGGING +#define TLSCONN_DEBUG(fmt, ...) \ + serverLog(LL_DEBUG, "TLSCONN: " fmt, __VA_ARGS__) +#else +#define TLSCONN_DEBUG(fmt, ...) +#endif + +ConnectionType CT_TLS; + +/* Normal socket connections have a simple events/handler correlation. + * + * With TLS connections we need to handle cases where during a logical read + * or write operation, the SSL library asks to block for the opposite + * socket operation. + * + * When this happens, we need to do two things: + * 1. Make sure we register for the even. + * 2. Make sure we know which handler needs to execute when the + * event fires. That is, if we notify the caller of a write operation + * that it blocks, and SSL asks for a read, we need to trigger the + * write handler again on the next read event. + * + */ + +typedef enum { + WANT_READ = 1, + WANT_WRITE +} WantIOType; + +#define TLS_CONN_FLAG_READ_WANT_WRITE (1<<0) +#define TLS_CONN_FLAG_WRITE_WANT_READ (1<<1) +#define TLS_CONN_FLAG_FD_SET (1<<2) + +typedef struct tls_connection { + connection c; + int flags; + SSL *ssl; + char *ssl_error; +} tls_connection; + +connection *connCreateTLS(void) { + tls_connection *conn = zcalloc(sizeof(tls_connection)); + conn->c.type = &CT_TLS; + conn->c.fd = -1; + conn->ssl = SSL_new(redis_tls_ctx); + return (connection *) conn; +} + +connection *connCreateAcceptedTLS(int fd, int require_auth) { + tls_connection *conn = (tls_connection *) connCreateTLS(); + conn->c.fd = fd; + conn->c.state = CONN_STATE_ACCEPTING; + + if (!require_auth) { + /* We still verify certificates if provided, but don't require them. + */ + SSL_set_verify(conn->ssl, SSL_VERIFY_PEER, NULL); + } + + SSL_set_fd(conn->ssl, conn->c.fd); + SSL_set_accept_state(conn->ssl); + + return (connection *) conn; +} + +static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask); + +/* Process the return code received from OpenSSL> + * Update the want parameter with expected I/O. + * Update the connection's error state if a real error has occured. + * Returns an SSL error code, or 0 if no further handling is required. + */ +static int handleSSLReturnCode(tls_connection *conn, int ret_value, WantIOType *want) { + if (ret_value <= 0) { + int ssl_err = SSL_get_error(conn->ssl, ret_value); + switch (ssl_err) { + case SSL_ERROR_WANT_WRITE: + *want = WANT_WRITE; + return 0; + case SSL_ERROR_WANT_READ: + *want = WANT_READ; + return 0; + case SSL_ERROR_SYSCALL: + conn->c.last_errno = errno; + if (conn->ssl_error) zfree(conn->ssl_error); + conn->ssl_error = errno ? zstrdup(strerror(errno)) : NULL; + break; + default: + /* Error! */ + conn->c.last_errno = 0; + if (conn->ssl_error) zfree(conn->ssl_error); + conn->ssl_error = zmalloc(512); + ERR_error_string_n(ERR_get_error(), conn->ssl_error, 512); + break; + } + + return ssl_err; + } + + return 0; +} + +void registerSSLEvent(tls_connection *conn, WantIOType want) { + int mask = aeGetFileEvents(server.el, conn->c.fd); + + switch (want) { + case WANT_READ: + if (mask & AE_WRITABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); + if (!(mask & AE_READABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, + tlsEventHandler, conn); + break; + case WANT_WRITE: + if (mask & AE_READABLE) aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); + if (!(mask & AE_WRITABLE)) aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, + tlsEventHandler, conn); + break; + default: + serverAssert(0); + break; + } +} + +void updateSSLEvent(tls_connection *conn) { + int mask = aeGetFileEvents(server.el, conn->c.fd); + int need_read = conn->c.read_handler || (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ); + int need_write = conn->c.write_handler || (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE); + + if (need_read && !(mask & AE_READABLE)) + aeCreateFileEvent(server.el, conn->c.fd, AE_READABLE, tlsEventHandler, conn); + if (!need_read && (mask & AE_READABLE)) + aeDeleteFileEvent(server.el, conn->c.fd, AE_READABLE); + + if (need_write && !(mask & AE_WRITABLE)) + aeCreateFileEvent(server.el, conn->c.fd, AE_WRITABLE, tlsEventHandler, conn); + if (!need_write && (mask & AE_WRITABLE)) + aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); +} + + +static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) { + UNUSED(el); + UNUSED(fd); + tls_connection *conn = clientData; + int ret; + + TLSCONN_DEBUG("tlsEventHandler(): fd=%d, state=%d, mask=%d, r=%d, w=%d, flags=%d", + fd, conn->c.state, mask, conn->c.read_handler != NULL, conn->c.write_handler != NULL, + conn->flags); + + ERR_clear_error(); + + switch (conn->c.state) { + case CONN_STATE_CONNECTING: + if (connGetSocketError((connection *) conn)) { + conn->c.last_errno = errno; + conn->c.state = CONN_STATE_ERROR; + } else { + if (!(conn->flags & TLS_CONN_FLAG_FD_SET)) { + SSL_set_fd(conn->ssl, conn->c.fd); + conn->flags |= TLS_CONN_FLAG_FD_SET; + } + ret = SSL_connect(conn->ssl); + if (ret <= 0) { + WantIOType want = 0; + if (!handleSSLReturnCode(conn, ret, &want)) { + registerSSLEvent(conn, want); + + /* Avoid hitting UpdateSSLEvent, which knows nothing + * of what SSL_connect() wants and instead looks at our + * R/W handlers. + */ + return; + } + + /* If not handled, it's an error */ + conn->c.state = CONN_STATE_ERROR; + } else { + conn->c.state = CONN_STATE_CONNECTED; + } + } + + if (!callHandler((connection *) conn, conn->c.conn_handler)) return; + conn->c.conn_handler = NULL; + break; + case CONN_STATE_ACCEPTING: + ret = SSL_accept(conn->ssl); + if (ret <= 0) { + WantIOType want = 0; + if (!handleSSLReturnCode(conn, ret, &want)) { + /* Avoid hitting UpdateSSLEvent, which knows nothing + * of what SSL_connect() wants and instead looks at our + * R/W handlers. + */ + registerSSLEvent(conn, want); + return; + } + + /* If not handled, it's an error */ + conn->c.state = CONN_STATE_ERROR; + } else { + conn->c.state = CONN_STATE_CONNECTED; + } + + if (!callHandler((connection *) conn, conn->c.conn_handler)) return; + conn->c.conn_handler = NULL; + break; + case CONN_STATE_CONNECTED: + if ((mask & AE_READABLE) && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)) { + conn->flags &= ~TLS_CONN_FLAG_WRITE_WANT_READ; + if (!callHandler((connection *) conn, conn->c.write_handler)) return; + } + + if ((mask & AE_WRITABLE) && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)) { + conn->flags &= ~TLS_CONN_FLAG_READ_WANT_WRITE; + if (!callHandler((connection *) conn, conn->c.read_handler)) return; + } + + if ((mask & AE_READABLE) && conn->c.read_handler) { + if (!callHandler((connection *) conn, conn->c.read_handler)) return; + } + + if ((mask & AE_WRITABLE) && conn->c.write_handler) { + if (!callHandler((connection *) conn, conn->c.write_handler)) return; + } + break; + default: + break; + } + + updateSSLEvent(conn); +} + +static void connTLSClose(connection *conn_) { + tls_connection *conn = (tls_connection *) conn_; + + if (conn->ssl) { + SSL_free(conn->ssl); + conn->ssl = NULL; + } + + if (conn->ssl_error) { + zfree(conn->ssl_error); + conn->ssl_error = NULL; + } + + CT_Socket.close(conn_); +} + +static int connTLSAccept(connection *_conn, ConnectionCallbackFunc accept_handler) { + tls_connection *conn = (tls_connection *) _conn; + int ret; + + if (conn->c.state != CONN_STATE_ACCEPTING) return C_ERR; + ERR_clear_error(); + + /* Try to accept */ + conn->c.conn_handler = accept_handler; + ret = SSL_accept(conn->ssl); + + if (ret <= 0) { + WantIOType want = 0; + if (!handleSSLReturnCode(conn, ret, &want)) { + registerSSLEvent(conn, want); /* We'll fire back */ + return C_OK; + } else { + conn->c.state = CONN_STATE_ERROR; + return C_ERR; + } + } + + conn->c.state = CONN_STATE_CONNECTED; + if (!callHandler((connection *) conn, conn->c.conn_handler)) return C_OK; + conn->c.conn_handler = NULL; + + return C_OK; +} + +static int connTLSConnect(connection *conn_, const char *addr, int port, const char *src_addr, ConnectionCallbackFunc connect_handler) { + tls_connection *conn = (tls_connection *) conn_; + + if (conn->c.state != CONN_STATE_NONE) return C_ERR; + ERR_clear_error(); + + /* Initiate Socket connection first */ + if (CT_Socket.connect(conn_, addr, port, src_addr, connect_handler) == C_ERR) return C_ERR; + + /* Return now, once the socket is connected we'll initiate + * TLS connection from the event handler. + */ + return C_OK; +} + +static int connTLSWrite(connection *conn_, const void *data, size_t data_len) { + tls_connection *conn = (tls_connection *) conn_; + int ret, ssl_err; + + if (conn->c.state != CONN_STATE_CONNECTED) return -1; + ERR_clear_error(); + ret = SSL_write(conn->ssl, data, data_len); + + if (ret <= 0) { + WantIOType want = 0; + if (!(ssl_err = handleSSLReturnCode(conn, ret, &want))) { + if (want == WANT_READ) conn->flags |= TLS_CONN_FLAG_WRITE_WANT_READ; + updateSSLEvent(conn); + errno = EAGAIN; + return -1; + } else { + if (ssl_err == SSL_ERROR_ZERO_RETURN || + ((ssl_err == SSL_ERROR_SYSCALL && !errno))) { + conn->c.state = CONN_STATE_CLOSED; + return 0; + } else { + conn->c.state = CONN_STATE_ERROR; + return -1; + } + } + } + + return ret; +} + +static int connTLSRead(connection *conn_, void *buf, size_t buf_len) { + tls_connection *conn = (tls_connection *) conn_; + int ret; + int ssl_err; + + if (conn->c.state != CONN_STATE_CONNECTED) return -1; + ERR_clear_error(); + ret = SSL_read(conn->ssl, buf, buf_len); + if (ret <= 0) { + WantIOType want = 0; + if (!(ssl_err = handleSSLReturnCode(conn, ret, &want))) { + if (want == WANT_WRITE) conn->flags |= TLS_CONN_FLAG_READ_WANT_WRITE; + updateSSLEvent(conn); + + errno = EAGAIN; + return -1; + } else { + if (ssl_err == SSL_ERROR_ZERO_RETURN || + ((ssl_err == SSL_ERROR_SYSCALL) && !errno)) { + conn->c.state = CONN_STATE_CLOSED; + return 0; + } else { + conn->c.state = CONN_STATE_ERROR; + return -1; + } + } + } + + return ret; +} + +static const char *connTLSGetLastError(connection *conn_) { + tls_connection *conn = (tls_connection *) conn_; + + if (conn->ssl_error) return conn->ssl_error; + return NULL; +} + +int connTLSSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { + conn->write_handler = func; + updateSSLEvent((tls_connection *) conn); + return C_OK; +} + +int connTLSSetReadHandler(connection *conn, ConnectionCallbackFunc func) { + conn->read_handler = func; + updateSSLEvent((tls_connection *) conn); + return C_OK; +} + +static void setBlockingTimeout(tls_connection *conn, long long timeout) { + anetBlock(NULL, conn->c.fd); + anetSendTimeout(NULL, conn->c.fd, timeout); + anetRecvTimeout(NULL, conn->c.fd, timeout); +} + +static void unsetBlockingTimeout(tls_connection *conn) { + anetNonBlock(NULL, conn->c.fd); + anetSendTimeout(NULL, conn->c.fd, 0); + anetRecvTimeout(NULL, conn->c.fd, 0); +} + +static int connTLSBlockingConnect(connection *conn_, const char *addr, int port, long long timeout) { + tls_connection *conn = (tls_connection *) conn_; + int ret; + + if (conn->c.state != CONN_STATE_NONE) return C_ERR; + + /* Initiate socket blocking connect first */ + if (CT_Socket.blocking_connect(conn_, addr, port, timeout) == C_ERR) return C_ERR; + + /* Initiate TLS connection now. We set up a send/recv timeout on the socket, + * which means the specified timeout will not be enforced accurately. */ + SSL_set_fd(conn->ssl, conn->c.fd); + setBlockingTimeout(conn, timeout); + + if ((ret = SSL_connect(conn->ssl)) <= 0) { + conn->c.state = CONN_STATE_ERROR; + return C_ERR; + } + unsetBlockingTimeout(conn); + + conn->c.state = CONN_STATE_CONNECTED; + return C_OK; +} + +static ssize_t connTLSSyncWrite(connection *conn_, char *ptr, ssize_t size, long long timeout) { + tls_connection *conn = (tls_connection *) conn_; + + setBlockingTimeout(conn, timeout); + SSL_clear_mode(conn->ssl, SSL_MODE_ENABLE_PARTIAL_WRITE); + int ret = SSL_write(conn->ssl, ptr, size); + SSL_set_mode(conn->ssl, SSL_MODE_ENABLE_PARTIAL_WRITE); + unsetBlockingTimeout(conn); + + return ret; +} + +static ssize_t connTLSSyncRead(connection *conn_, char *ptr, ssize_t size, long long timeout) { + tls_connection *conn = (tls_connection *) conn_; + + setBlockingTimeout(conn, timeout); + int ret = SSL_read(conn->ssl, ptr, size); + unsetBlockingTimeout(conn); + + return ret; +} + +static ssize_t connTLSSyncReadLine(connection *conn_, char *ptr, ssize_t size, long long timeout) { + tls_connection *conn = (tls_connection *) conn_; + ssize_t nread = 0; + + setBlockingTimeout(conn, timeout); + + size--; + while(size) { + char c; + + if (SSL_read(conn->ssl,&c,1) <= 0) { + nread = -1; + goto exit; + } + if (c == '\n') { + *ptr = '\0'; + if (nread && *(ptr-1) == '\r') *(ptr-1) = '\0'; + goto exit; + } else { + *ptr++ = c; + *ptr = '\0'; + nread++; + } + size--; + } +exit: + unsetBlockingTimeout(conn); + return nread; +} + +/* TODO: This is probably not the right thing to do, but as we handle proxying from child + * processes we'll probably not need any shutdown mechanism anyway so this is just a + * place holder for now. + */ +static int connTLSShutdown(connection *conn_, int how) { + UNUSED(how); + tls_connection *conn = (tls_connection *) conn_; + + return SSL_shutdown(conn->ssl); +} + +ConnectionType CT_TLS = { + .ae_handler = tlsEventHandler, + .accept = connTLSAccept, + .connect = connTLSConnect, + .blocking_connect = connTLSBlockingConnect, + .read = connTLSRead, + .write = connTLSWrite, + .close = connTLSClose, + .set_write_handler = connTLSSetWriteHandler, + .set_read_handler = connTLSSetReadHandler, + .get_last_error = connTLSGetLastError, + .sync_write = connTLSSyncWrite, + .sync_read = connTLSSyncRead, + .sync_readline = connTLSSyncReadLine, + .shutdown = connTLSShutdown +}; + +#else /* USE_OPENSSL */ + +void tlsInit(void) { +} + +int tlsConfigure(const char *cert_file, const char *key_file, + const char *dh_params_file, const char *ca_cert_file) { + UNUSED(cert_file); + UNUSED(key_file); + UNUSED(dh_params_file); + UNUSED(ca_cert_file); + return C_OK; +} + +int tlsConfigureServer(void) { + return C_OK; +} + +connection *connCreateTLS(void) { + return NULL; +} + +connection *connCreateAcceptedTLS(int fd, int require_auth) { + UNUSED(fd); + + return NULL; +} + +#endif diff --git a/tests/cluster/run.tcl b/tests/cluster/run.tcl index 93603ddc9..d9a7d7ee5 100644 --- a/tests/cluster/run.tcl +++ b/tests/cluster/run.tcl @@ -8,6 +8,7 @@ source ../instances.tcl source ../../support/cluster.tcl ; # Redis Cluster client. set ::instances_count 20 ; # How many instances we use at max. +set ::tlsdir "../../tls" proc main {} { parse_options diff --git a/tests/cluster/tests/04-resharding.tcl b/tests/cluster/tests/04-resharding.tcl index 68fba135e..33f861dc5 100644 --- a/tests/cluster/tests/04-resharding.tcl +++ b/tests/cluster/tests/04-resharding.tcl @@ -4,6 +4,7 @@ # are preseved across iterations. source "../tests/includes/init-tests.tcl" +source "../../../tests/support/cli.tcl" test "Create a 5 nodes cluster" { create_cluster 5 5 @@ -79,6 +80,7 @@ test "Cluster consistency during live resharding" { --cluster-to $target \ --cluster-slots 100 \ --cluster-yes \ + {*}[rediscli_tls_config "../../../tests"] \ | [info nameofexecutable] \ ../tests/helpers/onlydots.tcl \ &] 0] diff --git a/tests/cluster/tests/12-replica-migration-2.tcl b/tests/cluster/tests/12-replica-migration-2.tcl index 3d8b7b04b..dd18a979a 100644 --- a/tests/cluster/tests/12-replica-migration-2.tcl +++ b/tests/cluster/tests/12-replica-migration-2.tcl @@ -5,6 +5,7 @@ # other masters have slaves. source "../tests/includes/init-tests.tcl" +source "../../../tests/support/cli.tcl" # Create a cluster with 5 master and 15 slaves, to make sure there are no # empty masters and make rebalancing simpler to handle during the test. @@ -33,7 +34,9 @@ test "Resharding all the master #0 slots away from it" { set output [exec \ ../../../src/redis-cli --cluster rebalance \ 127.0.0.1:[get_instance_attrib redis 0 port] \ + {*}[rediscli_tls_config "../../../tests"] \ --cluster-weight ${master0_id}=0 >@ stdout ] + } test "Master #0 should lose its replicas" { @@ -51,6 +54,7 @@ test "Resharding back some slot to master #0" { set output [exec \ ../../../src/redis-cli --cluster rebalance \ 127.0.0.1:[get_instance_attrib redis 0 port] \ + {*}[rediscli_tls_config "../../../tests"] \ --cluster-weight ${master0_id}=.01 \ --cluster-use-empty-masters >@ stdout] } diff --git a/tests/helpers/bg_block_op.tcl b/tests/helpers/bg_block_op.tcl index 238d3874f..c8b323308 100644 --- a/tests/helpers/bg_block_op.tcl +++ b/tests/helpers/bg_block_op.tcl @@ -1,6 +1,8 @@ source tests/support/redis.tcl source tests/support/util.tcl +set ::tlsdir "tests/tls" + # This function sometimes writes sometimes blocking-reads from lists/sorted # sets. There are multiple processes like this executing at the same time # so that we have some chance to trap some corner condition if there is @@ -8,8 +10,8 @@ source tests/support/util.tcl # space to just a few elements, and balance the operations so that it is # unlikely that lists and zsets just get more data without ever causing # blocking. -proc bg_block_op {host port db ops} { - set r [redis $host $port] +proc bg_block_op {host port db ops tls} { + set r [redis $host $port 0 $tls] $r select $db for {set j 0} {$j < $ops} {incr j} { @@ -49,4 +51,4 @@ proc bg_block_op {host port db ops} { } } -bg_block_op [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] +bg_block_op [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] diff --git a/tests/helpers/bg_complex_data.tcl b/tests/helpers/bg_complex_data.tcl index dffd7c668..e888748a7 100644 --- a/tests/helpers/bg_complex_data.tcl +++ b/tests/helpers/bg_complex_data.tcl @@ -1,10 +1,12 @@ source tests/support/redis.tcl source tests/support/util.tcl -proc bg_complex_data {host port db ops} { - set r [redis $host $port] +set ::tlsdir "tests/tls" + +proc bg_complex_data {host port db ops tls} { + set r [redis $host $port 0 $tls] $r select $db createComplexDataset $r $ops } -bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] +bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] [lindex $argv 4] diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index 6d1a34516..fd6aad40c 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -1,8 +1,10 @@ source tests/support/redis.tcl -proc gen_write_load {host port seconds} { +set ::tlsdir "tests/tls" + +proc gen_write_load {host port seconds tls} { set start_time [clock seconds] - set r [redis $host $port 1] + set r [redis $host $port 0 $tls] $r select 9 while 1 { $r set [expr rand()] [expr rand()] @@ -12,4 +14,4 @@ proc gen_write_load {host port seconds} { } } -gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] +gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] diff --git a/tests/instances.tcl b/tests/instances.tcl index 357b34818..0a0cbab12 100644 --- a/tests/instances.tcl +++ b/tests/instances.tcl @@ -17,6 +17,7 @@ source ../support/test.tcl set ::verbose 0 set ::valgrind 0 +set ::tls 0 set ::pause_on_error 0 set ::simulate_error 0 set ::failed 0 @@ -69,7 +70,19 @@ proc spawn_instance {type base_port count {conf {}}} { # Write the instance config file. set cfgfile [file join $dirname $type.conf] set cfg [open $cfgfile w] - puts $cfg "port $port" + if {$::tls} { + puts $cfg "tls-port $port" + puts $cfg "tls-replication yes" + puts $cfg "tls-cluster yes" + puts $cfg "port 0" + puts $cfg [format "tls-cert-file %s/../../tls/redis.crt" [pwd]] + puts $cfg [format "tls-key-file %s/../../tls/redis.key" [pwd]] + puts $cfg [format "tls-dh-params-file %s/../../tls/redis.dh" [pwd]] + puts $cfg [format "tls-ca-cert-file %s/../../tls/ca.crt" [pwd]] + puts $cfg "loglevel debug" + } else { + puts $cfg "port $port" + } puts $cfg "dir ./$dirname" puts $cfg "logfile log.txt" # Add additional config files @@ -88,7 +101,7 @@ proc spawn_instance {type base_port count {conf {}}} { } # Push the instance into the right list - set link [redis 127.0.0.1 $port] + set link [redis 127.0.0.1 $port 0 $::tls] $link reconnect 1 lappend ::${type}_instances [list \ pid $pid \ @@ -148,6 +161,13 @@ proc parse_options {} { set ::simulate_error 1 } elseif {$opt eq {--valgrind}} { set ::valgrind 1 + } elseif {$opt eq {--tls}} { + package require tls 1.6 + ::tls::init \ + -cafile "$::tlsdir/ca.crt" \ + -certfile "$::tlsdir/redis.crt" \ + -keyfile "$::tlsdir/redis.key" + set ::tls 1 } elseif {$opt eq "--help"} { puts "Hello, I'm sentinel.tcl and I run Sentinel unit tests." puts "\nOptions:" @@ -492,7 +512,7 @@ proc restart_instance {type id} { } # Connect with it with a fresh link - set link [redis 127.0.0.1 $port] + set link [redis 127.0.0.1 $port 0 $::tls] $link reconnect 1 set_instance_attrib $type $id link $link diff --git a/tests/integration/aof-race.tcl b/tests/integration/aof-race.tcl index fb8d71083..2991e7962 100644 --- a/tests/integration/aof-race.tcl +++ b/tests/integration/aof-race.tcl @@ -13,8 +13,9 @@ tags {"aof"} { # cleaned after a child responsible for an AOF rewrite exited. This buffer # was subsequently appended to the new AOF, resulting in duplicate commands. start_server_aof [list dir $server_path] { - set client [redis [srv host] [srv port]] - set bench [open "|src/redis-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] + set client [redis [srv host] [srv port] 0 $::tls] + set bench [open "|src/redis-benchmark -q -s [srv unixsocket] -c 20 -n 20000 incr foo" "r+"] + after 100 # Benchmark should be running by now: start background rewrite @@ -29,7 +30,7 @@ tags {"aof"} { # Restart server to replay AOF start_server_aof [list dir $server_path] { - set client [redis [srv host] [srv port]] + set client [redis [srv host] [srv port] 0 $::tls] assert_equal 20000 [$client get foo] } } diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index e397faeeb..e276a6254 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -52,7 +52,7 @@ tags {"aof"} { assert_equal 1 [is_alive $srv] } - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] test "Truncated AOF loaded: we expect foo to be equal to 5" { assert {[$client get foo] eq "5"} @@ -69,7 +69,7 @@ tags {"aof"} { assert_equal 1 [is_alive $srv] } - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] test "Truncated AOF loaded: we expect foo to be equal to 6 now" { assert {[$client get foo] eq "6"} @@ -170,7 +170,7 @@ tags {"aof"} { } test "Fixed AOF: Keyspace should contain values that were parseable" { - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_for_condition 50 100 { [catch {$client ping} e] == 0 } else { @@ -194,7 +194,7 @@ tags {"aof"} { } test "AOF+SPOP: Set should have 1 member" { - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_for_condition 50 100 { [catch {$client ping} e] == 0 } else { @@ -218,7 +218,7 @@ tags {"aof"} { } test "AOF+SPOP: Set should have 1 member" { - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_for_condition 50 100 { [catch {$client ping} e] == 0 } else { @@ -241,7 +241,7 @@ tags {"aof"} { } test "AOF+EXPIRE: List should be empty" { - set client [redis [dict get $srv host] [dict get $srv port]] + set client [redis [dict get $srv host] [dict get $srv port] 0 $::tls] wait_for_condition 50 100 { [catch {$client ping} e] == 0 } else { diff --git a/tests/integration/block-repl.tcl b/tests/integration/block-repl.tcl index c111b805b..07eceb228 100644 --- a/tests/integration/block-repl.tcl +++ b/tests/integration/block-repl.tcl @@ -2,9 +2,9 @@ # Unlike stream operations such operations are "pop" style, so they consume # the list or sorted set, and must be replicated correctly. -proc start_bg_block_op {host port db ops} { +proc start_bg_block_op {host port db ops tls} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/bg_block_op.tcl $host $port $db $ops & + exec $tclsh tests/helpers/bg_block_op.tcl $host $port $db $ops $tls & } proc stop_bg_block_op {handle} { @@ -18,9 +18,9 @@ start_server {tags {"repl"}} { set master_port [srv -1 port] set slave [srv 0 client] - set load_handle0 [start_bg_block_op $master_host $master_port 9 100000] - set load_handle1 [start_bg_block_op $master_host $master_port 9 100000] - set load_handle2 [start_bg_block_op $master_host $master_port 9 100000] + set load_handle0 [start_bg_block_op $master_host $master_port 9 100000 $::tls] + set load_handle1 [start_bg_block_op $master_host $master_port 9 100000 $::tls] + set load_handle2 [start_bg_block_op $master_host $master_port 9 100000 $::tls] test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port diff --git a/tests/integration/psync2-reg.tcl b/tests/integration/psync2-reg.tcl index 3d408368e..b5ad021e2 100644 --- a/tests/integration/psync2-reg.tcl +++ b/tests/integration/psync2-reg.tcl @@ -18,6 +18,7 @@ start_server {} { set R($j) [srv [expr 0-$j] client] set R_host($j) [srv [expr 0-$j] host] set R_port($j) [srv [expr 0-$j] port] + set R_unixsocket($j) [srv [expr 0-$j] unixsocket] if {$debug_msg} {puts "Log file: [srv [expr 0-$j] stdout]"} } @@ -36,7 +37,7 @@ start_server {} { } set cycle_start_time [clock milliseconds] - set bench_pid [exec src/redis-benchmark -p $R_port(0) -n 10000000 -r 1000 incr __rand_int__ > /dev/null &] + set bench_pid [exec src/redis-benchmark -s $R_unixsocket(0) -n 10000000 -r 1000 incr __rand_int__ > /dev/null &] while 1 { set elapsed [expr {[clock milliseconds]-$cycle_start_time}] if {$elapsed > $duration*1000} break diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 40e4222e3..5d1635950 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -1,7 +1,10 @@ +source tests/support/cli.tcl + start_server {tags {"cli"}} { proc open_cli {} { set ::env(TERM) dumb - set fd [open [format "|src/redis-cli -p %d -n 9" [srv port]] "r+"] + set cmdline [rediscli [srv port] "-n 9"] + set fd [open "|$cmdline" "r+"] fconfigure $fd -buffering none fconfigure $fd -blocking false fconfigure $fd -translation binary @@ -54,8 +57,8 @@ start_server {tags {"cli"}} { } proc _run_cli {opts args} { - set cmd [format "src/redis-cli -p %d -n 9 $args" [srv port]] - foreach {key value} $opts { + set cmd [rediscli [srv port] [list -n 9 {*}$args]] + foreach {key value} $args { if {$key eq "pipe"} { set cmd "sh -c \"$value | $cmd\"" } diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 5d32555b0..43bc684b4 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -29,6 +29,9 @@ start_server {tags {"repl"}} { $slave slaveof $master_host $master_port test {Slave enters handshake} { + if {$::tls} { + fail "TLS with repl-diskless-sync not supported yet." + } wait_for_condition 50 1000 { [string match *handshake* [$slave role]] } else { @@ -184,6 +187,10 @@ start_server {tags {"repl"}} { } foreach mdl {no yes} { + if {$::tls && $mdl eq "yes"} { + puts "** Skipping test: TLS with repl-diskless-sync not supported yet." + continue + } foreach sdl {disabled swapdb} { start_server {tags {"repl"}} { set master [srv 0 client] @@ -320,6 +327,9 @@ start_server {tags {"repl"}} { } test {slave fails full sync and diskless load swapdb recoveres it} { + if {$::tls} { + fail "" + } start_server {tags {"repl"}} { set slave [srv 0 client] set slave_host [srv 0 host] @@ -387,6 +397,10 @@ test {slave fails full sync and diskless load swapdb recoveres it} { } test {diskless loading short read} { + if {$::tls} { + fail "TLS with repl-diskless-sync not supported yet." + } + start_server {tags {"repl"}} { set replica [srv 0 client] set replica_host [srv 0 host] diff --git a/tests/sentinel/tests/07-down-conditions.tcl b/tests/sentinel/tests/07-down-conditions.tcl index fb2993b6f..a12ea3151 100644 --- a/tests/sentinel/tests/07-down-conditions.tcl +++ b/tests/sentinel/tests/07-down-conditions.tcl @@ -1,6 +1,7 @@ # Test conditions where an instance is considered to be down source "../tests/includes/init-tests.tcl" +source "../../../tests/support/cli.tcl" proc ensure_master_up {} { wait_for_condition 1000 50 { @@ -28,7 +29,7 @@ test "Crash the majority of Sentinels to prevent failovers for this unit" { test "SDOWN is triggered by non-responding but not crashed instance" { lassign [S 4 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] host port ensure_master_up - exec ../../../src/redis-cli -h $host -p $port debug sleep 10 > /dev/null & + exec ../../../src/redis-cli -h $host -p $port {*}[rediscli_tls_config "../../../tests"] debug sleep 10 > /dev/null & ensure_master_down ensure_master_up } diff --git a/tests/support/cli.tcl b/tests/support/cli.tcl new file mode 100644 index 000000000..37c902a50 --- /dev/null +++ b/tests/support/cli.tcl @@ -0,0 +1,19 @@ +proc rediscli_tls_config {testsdir} { + set tlsdir [file join $testsdir tls] + set cert [file join $tlsdir redis.crt] + set key [file join $tlsdir redis.key] + set cacert [file join $tlsdir ca.crt] + + if {$::tls} { + return [list --tls --cert $cert --key $key --cacert $cacert] + } else { + return {} + } +} + +proc rediscli {port {opts {}}} { + set cmd [list src/redis-cli -p $port] + lappend cmd {*}[rediscli_tls_config "tests"] + lappend cmd {*}$opts + return $cmd +} diff --git a/tests/support/cluster.tcl b/tests/support/cluster.tcl index 1576053b4..74587e1f7 100644 --- a/tests/support/cluster.tcl +++ b/tests/support/cluster.tcl @@ -62,7 +62,7 @@ proc ::redis_cluster::__method__refresh_nodes_map {id} { lassign [split $ip_port :] start_host start_port if {[catch { set r {} - set r [redis $start_host $start_port] + set r [redis $start_host $start_port 0 $::tls] set nodes_descr [$r cluster nodes] $r close } e]} { @@ -107,7 +107,7 @@ proc ::redis_cluster::__method__refresh_nodes_map {id} { # Connect to the node set link {} - catch {set link [redis $host $port]} + catch {set link [redis $host $port 0 $::tls]} # Build this node description as an hash. set node [dict create \ diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index cd8ae3a34..be6a11c7a 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -39,8 +39,17 @@ array set ::redis::callback {} array set ::redis::state {} ;# State in non-blocking reply reading array set ::redis::statestack {} ;# Stack of states, for nested mbulks -proc redis {{server 127.0.0.1} {port 6379} {defer 0}} { - set fd [socket $server $port] +proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0}} { + if {$tls} { + package require tls + ::tls::init \ + -cafile "$::tlsdir/ca.crt" \ + -certfile "$::tlsdir/redis.crt" \ + -keyfile "$::tlsdir/redis.key" + set fd [::tls::socket $server $port] + } else { + set fd [socket $server $port] + } fconfigure $fd -translation binary set id [incr ::redis::id] set ::redis::fd($id) $fd @@ -48,6 +57,7 @@ proc redis {{server 127.0.0.1} {port 6379} {defer 0}} { set ::redis::blocking($id) 1 set ::redis::deferred($id) $defer set ::redis::reconnect($id) 0 + set ::redis::tls $tls ::redis::redis_reset_state $id interp alias {} ::redis::redisHandle$id {} ::redis::__dispatch__ $id } @@ -72,7 +82,11 @@ proc ::redis::__dispatch__raw__ {id method argv} { # Reconnect the link if needed. if {$fd eq {}} { lassign $::redis::addr($id) host port - set ::redis::fd($id) [socket $host $port] + if {$::redis::tls} { + set ::redis::fd($id) [::tls::socket $host $port] + } else { + set ::redis::fd($id) [socket $host $port] + } fconfigure $::redis::fd($id) -translation binary set fd $::redis::fd($id) } diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 0edb25d8a..b20f1ad36 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -92,7 +92,11 @@ proc is_alive config { proc ping_server {host port} { set retval 0 if {[catch { - set fd [socket $host $port] + if {$::tls} { + set fd [::tls::socket $host $port] + } else { + set fd [socket $host $port] + } fconfigure $fd -translation binary puts $fd "PING\r\n" flush $fd @@ -136,7 +140,6 @@ proc tags {tags code} { uplevel 1 $code set ::tags [lrange $::tags 0 end-[llength $tags]] } - proc start_server {options {code undefined}} { # If we are running against an external server, we just push the # host/port pair in the stack the first time @@ -145,7 +148,7 @@ proc start_server {options {code undefined}} { set srv {} dict set srv "host" $::host dict set srv "port" $::port - set client [redis $::host $::port] + set client [redis $::host $::port 0 $::tls] dict set srv "client" $client $client select 9 @@ -178,6 +181,13 @@ proc start_server {options {code undefined}} { set data [split [exec cat "tests/assets/$baseconfig"] "\n"] set config {} + if {$::tls} { + dict set config "tls-cert-file" [format "%s/tests/tls/redis.crt" [pwd]] + dict set config "tls-key-file" [format "%s/tests/tls/redis.key" [pwd]] + dict set config "tls-dh-params-file" [format "%s/tests/tls/redis.dh" [pwd]] + dict set config "tls-ca-cert-file" [format "%s/tests/tls/ca.crt" [pwd]] + dict set config "loglevel" "debug" + } foreach line $data { if {[string length $line] > 0 && [string index $line 0] ne "#"} { set elements [split $line " "] @@ -192,7 +202,17 @@ proc start_server {options {code undefined}} { # start every server on a different port set ::port [find_available_port [expr {$::port+1}]] - dict set config port $::port + if {$::tls} { + dict set config "port" 0 + dict set config "tls-port" $::port + dict set config "tls-cluster" "yes" + dict set config "tls-replication" "yes" + } else { + dict set config port $::port + } + + set unixsocket [file normalize [format "%s/%s" [dict get $config "dir"] "socket"]] + dict set config "unixsocket" $unixsocket # apply overrides from global space and arguments foreach {directive arguments} [concat $::global_overrides $overrides] { @@ -254,10 +274,11 @@ proc start_server {options {code undefined}} { } # setup properties to be able to initialize a client object + set port_param [expr $::tls ? {"tls-port"} : {"port"}] set host $::host set port $::port if {[dict exists $config bind]} { set host [dict get $config bind] } - if {[dict exists $config port]} { set port [dict get $config port] } + if {[dict exists $config $port_param]} { set port [dict get $config $port_param] } # setup config dict dict set srv "config_file" $config_file @@ -267,6 +288,7 @@ proc start_server {options {code undefined}} { dict set srv "port" $port dict set srv "stdout" $stdout dict set srv "stderr" $stderr + dict set srv "unixsocket" $unixsocket # if a block of code is supplied, we wait for the server to become # available, create a client object and kill the server afterwards diff --git a/tests/support/util.tcl b/tests/support/util.tcl index c2e76afad..7ecf5b79c 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -395,7 +395,7 @@ proc colorstr {color str} { # of seconds to the specified Redis instance. proc start_write_load {host port seconds} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds & + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds $::tls & } # Stop a process generating write load executed with start_write_load. @@ -423,7 +423,7 @@ proc lshuffle {list} { # of ops to the specified Redis instance. proc start_bg_complex_data {host port db ops} { set tclsh [info nameofexecutable] - exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops & + exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops $::tls & } # Stop a process generating write load executed with start_bg_complex_data. diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 1442067f5..cb7e4e328 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -63,6 +63,7 @@ set ::all_tests { unit/lazyfree unit/wait unit/pendingquerybuf + unit/tls } # Index to the next test to run in the ::all_tests list. set ::next_test 0 @@ -71,6 +72,7 @@ set ::host 127.0.0.1 set ::port 21111 set ::traceleaks 0 set ::valgrind 0 +set ::tls 0 set ::stack_logging 0 set ::verbose 0 set ::quiet 0 @@ -92,6 +94,7 @@ set ::dont_clean 0 set ::wait_server 0 set ::stop_on_failure 0 set ::loop 0 +set ::tlsdir "tests/tls" # Set to 1 when we are running in client mode. The Redis test uses a # server-client model to run tests simultaneously. The server instance @@ -146,7 +149,7 @@ proc reconnect {args} { set host [dict get $srv "host"] set port [dict get $srv "port"] set config [dict get $srv "config"] - set client [redis $host $port] + set client [redis $host $port 0 $::tls] dict set srv "client" $client # select the right db when we don't have to authenticate @@ -166,7 +169,7 @@ proc redis_deferring_client {args} { } # create client that defers reading reply - set client [redis [srv $level "host"] [srv $level "port"] 1] + set client [redis [srv $level "host"] [srv $level "port"] 1 $::tls] # select the right db and read the response (OK) $client select 9 @@ -204,7 +207,7 @@ proc test_server_main {} { if {!$::quiet} { puts "Starting test server at port $port" } - socket -server accept_test_clients -myaddr 127.0.0.1 $port + socket -server accept_test_clients -myaddr 127.0.0.1 $port # Start the client instances set ::clients_pids {} @@ -450,6 +453,7 @@ proc print_help_screen {} { "--stop Blocks once the first test fails." "--loop Execute the specified set of tests forever." "--wait-server Wait after server is started (so that you can attach a debugger)." + "--tls Run tests in TLS mode." "--help Print this help screen." } "\n"] } @@ -486,6 +490,13 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } elseif {$opt eq {--quiet}} { set ::quiet 1 + } elseif {$opt eq {--tls}} { + package require tls 1.6 + set ::tls 1 + ::tls::init \ + -cafile "$::tlsdir/ca.crt" \ + -certfile "$::tlsdir/redis.crt" \ + -keyfile "$::tlsdir/redis.key" } elseif {$opt eq {--host}} { set ::external 1 set ::host $arg @@ -565,7 +576,11 @@ if {[llength $::single_tests] > 0} { } proc attach_to_replication_stream {} { - set s [socket [srv 0 "host"] [srv 0 "port"]] + if {$::tls} { + set s [::tls::socket [srv 0 "host"] [srv 0 "port"]] + } else { + set s [socket [srv 0 "host"] [srv 0 "port"]] + } fconfigure $s -translation binary puts -nonewline $s "SYNC\r\n" flush $s diff --git a/tests/unit/limits.tcl b/tests/unit/limits.tcl index b37ea9b0f..38ba76208 100644 --- a/tests/unit/limits.tcl +++ b/tests/unit/limits.tcl @@ -1,4 +1,9 @@ start_server {tags {"limits"} overrides {maxclients 10}} { + if {$::tls} { + set expected_code "*I/O error*" + } else { + set expected_code "*ERR max*reached*" + } test {Check if maxclients works refusing connections} { set c 0 catch { @@ -12,5 +17,5 @@ start_server {tags {"limits"} overrides {maxclients 10}} { } e assert {$c > 8 && $c <= 10} set e - } {*ERR max*reached*} + } $expected_code } diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index 965902456..7720c055a 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -166,7 +166,11 @@ start_server {tags {"other"}} { tags {protocol} { test {PIPELINING stresser (also a regression for the old epoll bug)} { - set fd2 [socket $::host $::port] + if {$::tls} { + set fd2 [::tls::socket $::host $::port] + } else { + set fd2 [socket $::host $::port] + } fconfigure $fd2 -encoding binary -translation binary puts -nonewline $fd2 "SELECT 9\r\n" flush $fd2 diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index ac99c3abb..4dfdc6f59 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -72,7 +72,11 @@ start_server {tags {"protocol"}} { foreach seq [list "\x00" "*\x00" "$\x00"] { incr c test "Protocol desync regression test #$c" { - set s [socket [srv 0 host] [srv 0 port]] + if {$::tls} { + set s [::tls::socket [srv 0 host] [srv 0 port]] + } else { + set s [socket [srv 0 host] [srv 0 port]] + } puts -nonewline $s $seq set payload [string repeat A 1024]"\n" set test_start [clock seconds] diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl new file mode 100644 index 000000000..58acdb6a9 --- /dev/null +++ b/tests/unit/tls.tcl @@ -0,0 +1,25 @@ +start_server {tags {"tls"}} { + if {$::tls} { + package require tls + + test {TLS: Not accepting non-TLS connections on a TLS port} { + set s [redis [srv 0 host] [srv 0 port]] + catch {$s PING} e + set e + } {*I/O error*} + + test {TLS: Verify tls-auth-clients behaves as expected} { + set s [redis [srv 0 host] [srv 0 port]] + ::tls::import [$s channel] + catch {$s PING} e + assert_match {*error*} $e + + set resp [r CONFIG SET tls-auth-clients no] + + set s [redis [srv 0 host] [srv 0 port]] + ::tls::import [$s channel] + catch {$s PING} e + assert_match {PONG} $e + } {} + } +} diff --git a/tests/unit/wait.tcl b/tests/unit/wait.tcl index e2f5d2942..c9cfa6ed4 100644 --- a/tests/unit/wait.tcl +++ b/tests/unit/wait.tcl @@ -1,3 +1,5 @@ +source tests/support/cli.tcl + start_server {tags {"wait"}} { start_server {} { set slave [srv 0 client] @@ -31,7 +33,8 @@ start_server {} { } test {WAIT should not acknowledge 1 additional copy if slave is blocked} { - exec src/redis-cli -h $slave_host -p $slave_port debug sleep 5 > /dev/null 2> /dev/null & + set cmd [rediscli $slave_port "-h $slave_host debug sleep 5"] + exec {*}$cmd > /dev/null 2> /dev/null & after 1000 ;# Give redis-cli the time to execute the command. $master set foo 0 $master incr foo diff --git a/utils/gen-test-certs.sh b/utils/gen-test-certs.sh new file mode 100755 index 000000000..a46edc55a --- /dev/null +++ b/utils/gen-test-certs.sh @@ -0,0 +1,23 @@ +#!/bin/bash +mkdir -p tests/tls +openssl genrsa -out tests/tls/ca.key 4096 +openssl req \ + -x509 -new -nodes -sha256 \ + -key tests/tls/ca.key \ + -days 3650 \ + -subj '/O=Redis Test/CN=Certificate Authority' \ + -out tests/tls/ca.crt +openssl genrsa -out tests/tls/redis.key 2048 +openssl req \ + -new -sha256 \ + -key tests/tls/redis.key \ + -subj '/O=Redis Test/CN=Server' | \ + openssl x509 \ + -req -sha256 \ + -CA tests/tls/ca.crt \ + -CAkey tests/tls/ca.key \ + -CAserial tests/tls/ca.txt \ + -CAcreateserial \ + -days 365 \ + -out tests/tls/redis.crt +openssl dhparam -out tests/tls/redis.dh 2048 From 6fd5ff8c98ee38fd6bfb03406804c00c20db5225 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 11 Aug 2019 16:07:53 +0300 Subject: [PATCH 168/320] diskless replication rdb transfer uses pipe, and writes to sockets form the parent process. misc: - handle SSL_has_pending by iterating though these in beforeSleep, and setting timeout of 0 to aeProcessEvents - fix issue with epoll signaling EPOLLHUP and EPOLLERR only to the write handlers. (needed to detect the rdb pipe was closed) - add key-load-delay config for testing - trim connShutdown which is no longer needed - rioFdsetWrite -> rioFdWrite - simplified since there's no longer need to write to multiple FDs - don't detect rdb child exited (don't call wait3) until we detect the pipe is closed - Cleanup bad optimization from rio.c, add another one --- TLS.md | 21 +--- src/ae.c | 14 +++ src/ae.h | 2 + src/ae_epoll.c | 4 +- src/aof.c | 2 + src/config.c | 10 ++ src/connection.c | 5 - src/connection.h | 9 +- src/networking.c | 27 +++-- src/rdb.c | 192 ++++++------------------------ src/replication.c | 150 ++++++++++++++++++++++- src/rio.c | 146 ++++++++++------------- src/rio.h | 14 +-- src/server.c | 85 ++++++++----- src/server.h | 15 ++- src/tls.c | 71 ++++++++--- tests/integration/replication.tcl | 170 +++++++++++++++++++++++--- 17 files changed, 582 insertions(+), 355 deletions(-) diff --git a/TLS.md b/TLS.md index ee24a8df5..90ed08e7c 100644 --- a/TLS.md +++ b/TLS.md @@ -81,23 +81,6 @@ implementation details between TLS and TCP. difficult, but there are probably other good reasons to improve that part anyway. -5. A mechanism to re-trigger read callbacks for connections with unread buffers - (the case of reading partial TLS frames): - - a) Before sleep should iterate connections looking for those with a read handler, - SSL_pending() != 0 and no read event. - b) If found, trigger read handler for these conns. - c) After iteration if this state persists, epoll should be called in a way - that won't block so the process continues and this behave the same as a - level trigerred epoll. - -Replication ------------ - -Diskless master replication is broken, until child/parent connection proxying is -implemented. - - TLS Features ------------ @@ -119,6 +102,10 @@ most actions. This will need to be cleaned up for proper TLS support. The best approach is probably to migrate to hiredis async mode. +redis-cli +--------- +1. Support tls in --slave and --rdb + Others ------ diff --git a/src/ae.c b/src/ae.c index 53629ef77..54c0d994e 100644 --- a/src/ae.c +++ b/src/ae.c @@ -76,6 +76,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) { eventLoop->maxfd = -1; eventLoop->beforesleep = NULL; eventLoop->aftersleep = NULL; + eventLoop->flags = 0; if (aeApiCreate(eventLoop) == -1) goto err; /* Events with mask == AE_NONE are not set. So let's initialize the * vector with it. */ @@ -97,6 +98,14 @@ int aeGetSetSize(aeEventLoop *eventLoop) { return eventLoop->setsize; } +/* Tells the next iteration/s of the event processing to set timeout of 0. */ +void aeDontWait(aeEventLoop *eventLoop, int noWait) { + if (noWait) + eventLoop->flags |= AE_DONT_WAIT; + else + eventLoop->flags &= ~AE_DONT_WAIT; +} + /* Resize the maximum set size of the event loop. * If the requested set size is smaller than the current set size, but * there is already a file descriptor in use that is >= the requested @@ -406,6 +415,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) } } + if (eventLoop->flags & AE_DONT_WAIT) { + tv.tv_sec = tv.tv_usec = 0; + tvp = &tv; + } + /* Call the multiplexing API, will return only on timeout or when * some event fires. */ numevents = aeApiPoll(eventLoop, tvp); diff --git a/src/ae.h b/src/ae.h index 184fe3d1b..bbb43fe6e 100644 --- a/src/ae.h +++ b/src/ae.h @@ -106,6 +106,7 @@ typedef struct aeEventLoop { void *apidata; /* This is used for polling API specific data */ aeBeforeSleepProc *beforesleep; aeBeforeSleepProc *aftersleep; + int flags; } aeEventLoop; /* Prototypes */ @@ -128,5 +129,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); +void aeDontWait(aeEventLoop *eventLoop, int noWait); #endif diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 410aac70d..fa197297e 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -121,8 +121,8 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (e->events & EPOLLIN) mask |= AE_READABLE; if (e->events & EPOLLOUT) mask |= AE_WRITABLE; - if (e->events & EPOLLERR) mask |= AE_WRITABLE; - if (e->events & EPOLLHUP) mask |= AE_WRITABLE; + if (e->events & EPOLLERR) mask |= AE_WRITABLE|AE_READABLE; + if (e->events & EPOLLHUP) mask |= AE_WRITABLE|AE_READABLE; eventLoop->fired[j].fd = e->data.fd; eventLoop->fired[j].mask = mask; } diff --git a/src/aof.c b/src/aof.c index eed994bf2..ef34097ed 100644 --- a/src/aof.c +++ b/src/aof.c @@ -836,6 +836,8 @@ int loadAppendOnlyFile(char *filename) { freeFakeClientArgv(fakeClient); fakeClient->cmd = NULL; if (server.aof_load_truncated) valid_up_to = ftello(fp); + if (server.key_load_delay) + usleep(server.key_load_delay); } /* This point can only be reached when EOF is reached without errors. diff --git a/src/config.c b/src/config.c index 456fb0226..792e45057 100644 --- a/src/config.c +++ b/src/config.c @@ -522,6 +522,12 @@ void loadServerConfigFromString(char *config) { err = "rdb-key-save-delay can't be negative"; goto loaderr; } + } else if (!strcasecmp(argv[0],"key-load-delay") && argc==2) { + server.key_load_delay = atoi(argv[1]); + if (server.key_load_delay < 0) { + err = "key-load-delay can't be negative"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; @@ -1191,6 +1197,8 @@ void configSetCommand(client *c) { "replica-priority",server.slave_priority,0,INT_MAX) { } config_set_numerical_field( "rdb-key-save-delay",server.rdb_key_save_delay,0,LLONG_MAX) { + } config_set_numerical_field( + "key-load-delay",server.key_load_delay,0,LLONG_MAX) { } config_set_numerical_field( "slave-announce-port",server.slave_announce_port,0,65535) { } config_set_numerical_field( @@ -1452,6 +1460,7 @@ void configGetCommand(client *c) { config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); config_get_numerical_field("rdb-key-save-delay",server.rdb_key_save_delay); + config_get_numerical_field("key-load-delay",server.key_load_delay); config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); /* Bool (yes/no) values */ @@ -2272,6 +2281,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); rewriteConfigNumericalOption(state,"rdb-key-save-delay",server.rdb_key_save_delay,CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY); + rewriteConfigNumericalOption(state,"key-load-delay",server.key_load_delay,CONFIG_DEFAULT_KEY_LOAD_DELAY); rewriteConfigStringOption(state,"tls-cert-file",server.tls_cert_file,NULL); rewriteConfigStringOption(state,"tls-key-file",server.tls_key_file,NULL); rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_dh_params_file,NULL); diff --git a/src/connection.c b/src/connection.c index 62c6f3506..601f175bc 100644 --- a/src/connection.c +++ b/src/connection.c @@ -140,10 +140,6 @@ void *connGetPrivateData(connection *conn) { * move here as we implement additional connection types. */ -static int connSocketShutdown(connection *conn, int how) { - return shutdown(conn->fd, how); -} - /* Close the connection and free resources. */ static void connSocketClose(connection *conn) { if (conn->fd != -1) { @@ -298,7 +294,6 @@ static ssize_t connSocketSyncReadLine(connection *conn, char *ptr, ssize_t size, ConnectionType CT_Socket = { .ae_handler = connSocketEventHandler, .close = connSocketClose, - .shutdown = connSocketShutdown, .write = connSocketWrite, .read = connSocketRead, .accept = connSocketAccept, diff --git a/src/connection.h b/src/connection.h index e3e844f95..3fdcddebe 100644 --- a/src/connection.h +++ b/src/connection.h @@ -55,7 +55,6 @@ typedef struct ConnectionType { int (*connect)(struct connection *conn, const char *addr, int port, const char *source_addr, ConnectionCallbackFunc connect_handler); int (*write)(struct connection *conn, const void *data, size_t data_len); int (*read)(struct connection *conn, void *buf, size_t buf_len); - int (*shutdown)(struct connection *conn, int how); void (*close)(struct connection *conn); int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler); @@ -159,10 +158,6 @@ static inline void connClose(connection *conn) { conn->type->close(conn); } -static inline int connShutdown(connection *conn, int how) { - return conn->type->shutdown(conn, how); -} - /* Returns the last error encountered by the connection, as a string. If no error, * a NULL is returned. */ @@ -208,4 +203,8 @@ int connFormatPeer(connection *conn, char *buf, size_t buf_len); int connSockName(connection *conn, char *ip, size_t ip_len, int *port); const char *connGetInfo(connection *conn, char *buf, size_t buf_len); +/* Helpers for tls special considerations */ +int tlsHasPendingData(); +void tlsProcessPendingData(); + #endif /* __REDIS_CONNECTION_H */ diff --git a/src/networking.c b/src/networking.c index 2d00b0e74..d52eb1eba 100644 --- a/src/networking.c +++ b/src/networking.c @@ -881,7 +881,7 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip) { serverLog(LL_WARNING, "Error accepting a client connection: %s (conn: %s)", connGetLastError(conn), connGetInfo(conn, conninfo, sizeof(conninfo))); - connClose(conn); + freeClient(connGetPrivateData(conn)); return; } } @@ -984,14 +984,21 @@ void unlinkClient(client *c) { c->client_list_node = NULL; } - /* In the case of diskless replication the fork is writing to the - * sockets and just closing the fd isn't enough, if we don't also - * shutdown the socket the fork will continue to write to the slave - * and the salve will only find out that it was disconnected when - * it will finish reading the rdb. */ - int need_shutdown = ((c->flags & CLIENT_SLAVE) && - (c->replstate == SLAVE_STATE_WAIT_BGSAVE_END)); - if (need_shutdown) connShutdown(c->conn, SHUT_RDWR); + /* Check if this is a replica waiting for diskless replication (rdb pipe), + * in which case it needs to be cleaned from that list */ + if (c->flags & CLIENT_SLAVE && + c->replstate == SLAVE_STATE_WAIT_BGSAVE_END && + server.rdb_pipe_conns) + { + int i; + for (i=0; i < server.rdb_pipe_numconns; i++) { + if (server.rdb_pipe_conns[i] == c->conn) { + rdbPipeWriteHandlerConnRemoved(c->conn); + server.rdb_pipe_conns[i] = NULL; + break; + } + } + } connClose(c->conn); c->conn = NULL; } @@ -1309,7 +1316,7 @@ int handleClientsWithPendingWrites(void) { { ae_flags |= AE_BARRIER; } - /* TODO: Handle write barriers in connection */ + /* TODO: Handle write barriers in connection (also see tlsProcessPendingData) */ if (connSetWriteHandler(c->conn, sendReplyToClient) == C_ERR) { freeClientAsync(c); } diff --git a/src/rdb.c b/src/rdb.c index 3b98f57bb..59f22916b 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2211,6 +2211,8 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { * own reference. */ decrRefCount(key); } + if (server.key_load_delay) + usleep(server.key_load_delay); /* Reset the state that is key-specified and is populated by * opcodes before the key, so that we start from scratch again. */ @@ -2306,8 +2308,6 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) { * This function covers the case of RDB -> Salves socket transfers for * diskless replication. */ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { - uint64_t *ok_slaves; - if (!bysignal && exitcode == 0) { serverLog(LL_NOTICE, "Background RDB transfer terminated with success"); @@ -2321,79 +2321,6 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { server.rdb_child_type = RDB_CHILD_TYPE_NONE; server.rdb_save_time_start = -1; - /* If the child returns an OK exit code, read the set of slave client - * IDs and the associated status code. We'll terminate all the slaves - * in error state. - * - * If the process returned an error, consider the list of slaves that - * can continue to be empty, so that it's just a special case of the - * normal code path. */ - ok_slaves = zmalloc(sizeof(uint64_t)); /* Make space for the count. */ - ok_slaves[0] = 0; - if (!bysignal && exitcode == 0) { - int readlen = sizeof(uint64_t); - - if (read(server.rdb_pipe_read_result_from_child, ok_slaves, readlen) == - readlen) - { - readlen = ok_slaves[0]*sizeof(uint64_t)*2; - - /* Make space for enough elements as specified by the first - * uint64_t element in the array. */ - ok_slaves = zrealloc(ok_slaves,sizeof(uint64_t)+readlen); - if (readlen && - read(server.rdb_pipe_read_result_from_child, ok_slaves+1, - readlen) != readlen) - { - ok_slaves[0] = 0; - } - } - } - - close(server.rdb_pipe_read_result_from_child); - close(server.rdb_pipe_write_result_to_parent); - - /* We can continue the replication process with all the slaves that - * correctly received the full payload. Others are terminated. */ - listNode *ln; - listIter li; - - listRewind(server.slaves,&li); - while((ln = listNext(&li))) { - client *slave = ln->value; - - if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) { - uint64_t j; - int errorcode = 0; - - /* Search for the slave ID in the reply. In order for a slave to - * continue the replication process, we need to find it in the list, - * and it must have an error code set to 0 (which means success). */ - for (j = 0; j < ok_slaves[0]; j++) { - if (slave->id == ok_slaves[2*j+1]) { - errorcode = ok_slaves[2*j+2]; - break; /* Found in slaves list. */ - } - } - if (j == ok_slaves[0] || errorcode != 0) { - serverLog(LL_WARNING, - "Closing slave %s: child->slave RDB transfer failed: %s", - replicationGetSlaveName(slave), - (errorcode == 0) ? "RDB transfer child aborted" - : strerror(errorcode)); - freeClient(slave); - } else { - serverLog(LL_WARNING, - "Slave %s correctly received the streamed RDB file.", - replicationGetSlaveName(slave)); - /* Restore the socket as non-blocking. */ - connNonBlock(slave->conn); - connSendTimeout(slave->conn,0); - } - } - } - zfree(ok_slaves); - updateSlavesWaitingBgsave((!bysignal && exitcode == 0) ? C_OK : C_ERR, RDB_CHILD_TYPE_SOCKET); } @@ -2425,9 +2352,6 @@ void killRDBChild(void) { /* Spawn an RDB child that writes the RDB to the sockets of the slaves * that are currently in SLAVE_STATE_WAIT_BGSAVE_START state. */ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { - connection **conns; - uint64_t *clientids; - int numconns; listNode *ln; listIter li; pid_t childpid; @@ -2436,35 +2360,30 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { if (server.aof_child_pid != -1 || server.rdb_child_pid != -1) return C_ERR; - /* Before to fork, create a pipe that will be used in order to - * send back to the parent the IDs of the slaves that successfully - * received all the writes. */ + /* Even if the previous fork child exited, don't start a new one until we + * drained the pipe. */ + if (server.rdb_pipe_conns) return C_ERR; + + /* Before to fork, create a pipe that is used to transfer the rdb bytes to + * the parant, we can't let it write directly to the sockets, since in case + * of TLS we must let the parent handle a contineous TLS state when the + * child terminates and parent takes over. */ if (pipe(pipefds) == -1) return C_ERR; - server.rdb_pipe_read_result_from_child = pipefds[0]; - server.rdb_pipe_write_result_to_parent = pipefds[1]; + server.rdb_pipe_read = pipefds[0]; + server.rdb_pipe_write = pipefds[1]; + anetNonBlock(NULL, server.rdb_pipe_read); - /* Collect the file descriptors of the slaves we want to transfer + /* Collect the connections of the replicas we want to transfer * the RDB to, which are i WAIT_BGSAVE_START state. */ - conns = zmalloc(sizeof(connection *)*listLength(server.slaves)); - /* We also allocate an array of corresponding client IDs. This will - * be useful for the child process in order to build the report - * (sent via unix pipe) that will be sent to the parent. */ - clientids = zmalloc(sizeof(uint64_t)*listLength(server.slaves)); - numconns = 0; - + server.rdb_pipe_conns = zmalloc(sizeof(connection *)*listLength(server.slaves)); + server.rdb_pipe_numconns = 0; + server.rdb_pipe_numconns_writing = 0; listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = ln->value; - if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_START) { - clientids[numconns] = slave->id; - conns[numconns++] = slave->conn; + server.rdb_pipe_conns[server.rdb_pipe_numconns++] = slave->conn; replicationSetupSlaveForFullResync(slave,getPsyncInitialOffset()); - /* Put the socket in blocking mode to simplify RDB transfer. - * We'll restore it when the children returns (since duped socket - * will share the O_NONBLOCK attribute with the parent). */ - connBlock(slave->conn); - connSendTimeout(slave->conn,server.repl_timeout*1000); } } @@ -2474,16 +2393,15 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { if ((childpid = fork()) == 0) { /* Child */ int retval; - rio slave_sockets; + rio rdb; - rioInitWithConnset(&slave_sockets,conns,numconns); - zfree(conns); + rioInitWithFd(&rdb,server.rdb_pipe_write); closeListeningSockets(0); redisSetProcTitle("redis-rdb-to-slaves"); - retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi); - if (retval == C_OK && rioFlush(&slave_sockets) == 0) + retval = rdbSaveRioWithEOFMark(&rdb,NULL,rsi); + if (retval == C_OK && rioFlush(&rdb) == 0) retval = C_ERR; if (retval == C_OK) { @@ -2497,48 +2415,9 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { server.child_info_data.cow_size = private_dirty; sendChildInfo(CHILD_INFO_TYPE_RDB); - - /* If we are returning OK, at least one slave was served - * with the RDB file as expected, so we need to send a report - * to the parent via the pipe. The format of the message is: - * - * ... - * - * len, slave IDs, and slave errors, are all uint64_t integers, - * so basically the reply is composed of 64 bits for the len field - * plus 2 additional 64 bit integers for each entry, for a total - * of 'len' entries. - * - * The 'id' represents the slave's client ID, so that the master - * can match the report with a specific slave, and 'error' is - * set to 0 if the replication process terminated with a success - * or the error code if an error occurred. */ - void *msg = zmalloc(sizeof(uint64_t)*(1+2*numconns)); - uint64_t *len = msg; - uint64_t *ids = len+1; - int j, msglen; - - *len = numconns; - for (j = 0; j < numconns; j++) { - *ids++ = clientids[j]; - *ids++ = slave_sockets.io.connset.state[j]; - } - - /* Write the message to the parent. If we have no good slaves or - * we are unable to transfer the message to the parent, we exit - * with an error so that the parent will abort the replication - * process with all the childre that were waiting. */ - msglen = sizeof(uint64_t)*(1+2*numconns); - if (*len == 0 || - write(server.rdb_pipe_write_result_to_parent,msg,msglen) - != msglen) - { - retval = C_ERR; - } - zfree(msg); } - zfree(clientids); - rioFreeConnset(&slave_sockets); + rioFreeFd(&rdb); + close(server.rdb_pipe_write); /* wake up the reader, tell it we're done. */ exitFromChild((retval == C_OK) ? 0 : 1); } else { /* Parent */ @@ -2552,17 +2431,16 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { listRewind(server.slaves,&li); while((ln = listNext(&li))) { client *slave = ln->value; - int j; - - for (j = 0; j < numconns; j++) { - if (slave->id == clientids[j]) { - slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START; - break; - } + if (slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END) { + slave->replstate = SLAVE_STATE_WAIT_BGSAVE_START; } } - close(pipefds[0]); - close(pipefds[1]); + close(server.rdb_pipe_write); + close(server.rdb_pipe_read); + zfree(server.rdb_pipe_conns); + server.rdb_pipe_conns = NULL; + server.rdb_pipe_numconns = 0; + server.rdb_pipe_numconns_writing = 0; closeChildInfoPipe(); } else { server.stat_fork_time = ustime()-start; @@ -2574,10 +2452,12 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; server.rdb_child_type = RDB_CHILD_TYPE_SOCKET; + close(server.rdb_pipe_write); /* close write in parent so that it can detect the close on the child. */ + if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) { + serverPanic("Unrecoverable error creating server.rdb_pipe_read file event."); + } updateDictResizePolicy(); } - zfree(clientids); - zfree(conns); return (childpid == -1) ? C_ERR : C_OK; } return C_OK; /* Unreached. */ diff --git a/src/replication.c b/src/replication.c index cbc52cb2e..9f5c19053 100644 --- a/src/replication.c +++ b/src/replication.c @@ -884,7 +884,7 @@ void sendBulkToSlave(connection *conn) { nwritten = connWrite(conn,slave->replpreamble,sdslen(slave->replpreamble)); if (nwritten == -1) { serverLog(LL_VERBOSE,"Write error sending RDB preamble to replica: %s", - strerror(errno)); + connGetLastError(conn)); freeClient(slave); return; } @@ -911,7 +911,7 @@ void sendBulkToSlave(connection *conn) { if ((nwritten = connWrite(conn,buf,buflen)) == -1) { if (connGetState(conn) != CONN_STATE_CONNECTED) { serverLog(LL_WARNING,"Write error sending DB to replica: %s", - strerror(errno)); + connGetLastError(conn)); freeClient(slave); } return; @@ -926,6 +926,152 @@ void sendBulkToSlave(connection *conn) { } } +/* Remove one write handler from the list of connections waiting to be writable + * during rdb pipe transfer. */ +void rdbPipeWriteHandlerConnRemoved(struct connection *conn) { + if (!connHasWriteHandler(conn)) + return; + connSetWriteHandler(conn, NULL); + server.rdb_pipe_numconns_writing--; + /* if there are no more writes for now for this conn, or write error: */ + if (server.rdb_pipe_numconns_writing == 0) { + if (aeCreateFileEvent(server.el, server.rdb_pipe_read, AE_READABLE, rdbPipeReadHandler,NULL) == AE_ERR) { + serverPanic("Unrecoverable error creating server.rdb_pipe_read file event."); + } + } +} + +/* Called in diskless master during transfer of data from the rdb pipe, when + * the replica becomes writable again. */ +void rdbPipeWriteHandler(struct connection *conn) { + serverAssert(server.rdb_pipe_bufflen>0); + client *slave = connGetPrivateData(conn); + int nwritten; + if ((nwritten = connWrite(conn, server.rdb_pipe_buff + slave->repldboff, + server.rdb_pipe_bufflen - slave->repldboff)) == -1) + { + if (connGetState(conn) == CONN_STATE_CONNECTED) + return; /* equivalent to EAGAIN */ + serverLog(LL_WARNING,"Write error sending DB to replica: %s", + connGetLastError(conn)); + freeClient(slave); + return; + } else { + slave->repldboff += nwritten; + server.stat_net_output_bytes += nwritten; + if (slave->repldboff < server.rdb_pipe_bufflen) + return; /* more data to write.. */ + } + rdbPipeWriteHandlerConnRemoved(conn); +} + +/* When the the pipe serving diskless rdb transfer is drained (write end was + * closed), we can clean up all the temporary variables, and cleanup after the + * fork child. */ +void RdbPipeCleanup() { + close(server.rdb_pipe_read); + zfree(server.rdb_pipe_conns); + server.rdb_pipe_conns = NULL; + server.rdb_pipe_numconns = 0; + server.rdb_pipe_numconns_writing = 0; + zfree(server.rdb_pipe_buff); + server.rdb_pipe_buff = NULL; + server.rdb_pipe_bufflen = 0; + + /* Since we're avoiding to detect the child exited as long as the pipe is + * not drained, so now is the time to check. */ + checkChildrenDone(); +} + +/* Called in diskless master, when there's data to read from the child's rdb pipe */ +void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask) { + UNUSED(mask); + UNUSED(clientData); + UNUSED(eventLoop); + int i; + if (!server.rdb_pipe_buff) + server.rdb_pipe_buff = zmalloc(PROTO_IOBUF_LEN); + serverAssert(server.rdb_pipe_numconns_writing==0); + + while (1) { + server.rdb_pipe_bufflen = read(fd, server.rdb_pipe_buff, PROTO_IOBUF_LEN); + if (server.rdb_pipe_bufflen < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) + return; + serverLog(LL_WARNING,"Diskless rdb transfer, read error sending DB to replicas: %s", strerror(errno)); + for (i=0; i < server.rdb_pipe_numconns; i++) { + connection *conn = server.rdb_pipe_conns[i]; + if (!conn) + continue; + client *slave = connGetPrivateData(conn); + freeClient(slave); + server.rdb_pipe_conns[i] = NULL; + } + killRDBChild(); + return; + } + + if (server.rdb_pipe_bufflen == 0) { + /* EOF - write end was closed. */ + int stillUp = 0; + aeDeleteFileEvent(server.el, server.rdb_pipe_read, AE_READABLE); + for (i=0; i < server.rdb_pipe_numconns; i++) + { + connection *conn = server.rdb_pipe_conns[i]; + if (!conn) + continue; + stillUp++; + } + serverLog(LL_WARNING,"Diskless rdb transfer, done reading from pipe, %d replicas still up.", stillUp); + RdbPipeCleanup(); + return; + } + + int stillAlive = 0; + for (i=0; i < server.rdb_pipe_numconns; i++) + { + int nwritten; + connection *conn = server.rdb_pipe_conns[i]; + if (!conn) + continue; + + client *slave = connGetPrivateData(conn); + if ((nwritten = connWrite(conn, server.rdb_pipe_buff, server.rdb_pipe_bufflen)) == -1) { + if (connGetState(conn) != CONN_STATE_CONNECTED) { + serverLog(LL_WARNING,"Diskless rdb transfer, write error sending DB to replica: %s", + connGetLastError(conn)); + freeClient(slave); + server.rdb_pipe_conns[i] = NULL; + continue; + } + /* An error and still in connected state, is equivalent to EAGAIN */ + slave->repldboff = 0; + } else { + slave->repldboff = nwritten; + server.stat_net_output_bytes += nwritten; + } + /* If we were unable to write all the data to one of the replicas, + * setup write handler (and disable pipe read handler, below) */ + if (nwritten != server.rdb_pipe_bufflen) { + server.rdb_pipe_numconns_writing++; + connSetWriteHandler(conn, rdbPipeWriteHandler); + } + stillAlive++; + } + + if (stillAlive == 0) { + serverLog(LL_WARNING,"Diskless rdb transfer, last replica dropped, killing fork child."); + killRDBChild(); + RdbPipeCleanup(); + } + /* Remove the pipe read handler if at least one write handler was set. */ + if (server.rdb_pipe_numconns_writing || stillAlive == 0) { + aeDeleteFileEvent(server.el, server.rdb_pipe_read, AE_READABLE); + break; + } + } +} + /* This function is called at the end of every background saving, * or when the replication RDB transfer strategy is modified from * disk to socket or the other way around. diff --git a/src/rio.c b/src/rio.c index 5e3b4a06e..c8c924380 100644 --- a/src/rio.c +++ b/src/rio.c @@ -272,85 +272,67 @@ void rioFreeConn(rio *r, sds *remaining) { r->io.conn.buf = NULL; } -/* ------------------- File descriptors set implementation ------------------ - * This target is used to write the RDB file to N different replicas via - * sockets, when the master just streams the data to the replicas without - * creating an RDB on-disk image (diskless replication option). +/* ------------------- File descriptor implementation ------------------ + * This target is used to write the RDB file to pipe, when the master just + * streams the data to the replicas without creating an RDB on-disk image + * (diskless replication option). * It only implements writes. */ /* Returns 1 or 0 for success/failure. - * The function returns success as long as we are able to correctly write - * to at least one file descriptor. * * When buf is NULL and len is 0, the function performs a flush operation * if there is some pending buffer, so this function is also used in order - * to implement rioFdsetFlush(). */ -static size_t rioFdsetWrite(rio *r, const void *buf, size_t len) { + * to implement rioFdFlush(). */ +static size_t rioFdWrite(rio *r, const void *buf, size_t len) { ssize_t retval; - int j; unsigned char *p = (unsigned char*) buf; int doflush = (buf == NULL && len == 0); - /* To start we always append to our buffer. If it gets larger than - * a given size, we actually write to the sockets. */ - if (len) { - r->io.connset.buf = sdscatlen(r->io.connset.buf,buf,len); - len = 0; /* Prevent entering the while below if we don't flush. */ - if (sdslen(r->io.connset.buf) > PROTO_IOBUF_LEN) doflush = 1; - } - - if (doflush) { - p = (unsigned char*) r->io.connset.buf; - len = sdslen(r->io.connset.buf); - } - - /* Write in little chunchs so that when there are big writes we - * parallelize while the kernel is sending data in background to - * the TCP socket. */ - while(len) { - size_t count = len < 1024 ? len : 1024; - int broken = 0; - for (j = 0; j < r->io.connset.numconns; j++) { - if (r->io.connset.state[j] != 0) { - /* Skip FDs alraedy in error. */ - broken++; - continue; - } - - /* Make sure to write 'count' bytes to the socket regardless - * of short writes. */ - size_t nwritten = 0; - while(nwritten != count) { - retval = connWrite(r->io.connset.conns[j],p+nwritten,count-nwritten); - if (retval <= 0) { - /* With blocking sockets, which is the sole user of this - * rio target, EWOULDBLOCK is returned only because of - * the SO_SNDTIMEO socket option, so we translate the error - * into one more recognizable by the user. */ - if (retval == -1 && errno == EWOULDBLOCK) errno = ETIMEDOUT; - break; - } - nwritten += retval; - } - - if (nwritten != count) { - /* Mark this FD as broken. */ - r->io.connset.state[j] = errno; - if (r->io.connset.state[j] == 0) r->io.connset.state[j] = EIO; - } + /* For small writes, we rather keep the data in user-space buffer, and flush + * it only when it grows. however for larger writes, we prefer to flush + * any pre-existing buffer, and write the new one directly without reallocs + * and memory copying. */ + if (len > PROTO_IOBUF_LEN) { + /* First, flush any pre-existing buffered data. */ + if (sdslen(r->io.fd.buf)) { + if (rioFdWrite(r, NULL, 0) == 0) + return 0; } - if (broken == r->io.connset.numconns) return 0; /* All the FDs in error. */ - p += count; - len -= count; - r->io.connset.pos += count; + /* Write the new data, keeping 'p' and 'len' from the input. */ + } else { + if (len) { + r->io.fd.buf = sdscatlen(r->io.fd.buf,buf,len); + if (sdslen(r->io.fd.buf) > PROTO_IOBUF_LEN) + doflush = 1; + if (!doflush) + return 1; + } + /* Flusing the buffered data. set 'p' and 'len' accordintly. */ + p = (unsigned char*) r->io.fd.buf; + len = sdslen(r->io.fd.buf); } - if (doflush) sdsclear(r->io.connset.buf); + size_t nwritten = 0; + while(nwritten != len) { + retval = write(r->io.fd.fd,p+nwritten,len-nwritten); + if (retval <= 0) { + /* With blocking io, which is the sole user of this + * rio target, EWOULDBLOCK is returned only because of + * the SO_SNDTIMEO socket option, so we translate the error + * into one more recognizable by the user. */ + if (retval == -1 && errno == EWOULDBLOCK) errno = ETIMEDOUT; + return 0; /* error. */ + } + nwritten += retval; + } + + r->io.fd.pos += len; + sdsclear(r->io.fd.buf); return 1; } /* Returns 1 or 0 for success/failure. */ -static size_t rioFdsetRead(rio *r, void *buf, size_t len) { +static size_t rioFdRead(rio *r, void *buf, size_t len) { UNUSED(r); UNUSED(buf); UNUSED(len); @@ -358,23 +340,23 @@ static size_t rioFdsetRead(rio *r, void *buf, size_t len) { } /* Returns read/write position in file. */ -static off_t rioFdsetTell(rio *r) { - return r->io.connset.pos; +static off_t rioFdTell(rio *r) { + return r->io.fd.pos; } /* Flushes any buffer to target device if applicable. Returns 1 on success * and 0 on failures. */ -static int rioFdsetFlush(rio *r) { +static int rioFdFlush(rio *r) { /* Our flush is implemented by the write method, that recognizes a * buffer set to NULL with a count of zero as a flush request. */ - return rioFdsetWrite(r,NULL,0); + return rioFdWrite(r,NULL,0); } -static const rio rioFdsetIO = { - rioFdsetRead, - rioFdsetWrite, - rioFdsetTell, - rioFdsetFlush, +static const rio rioFdIO = { + rioFdRead, + rioFdWrite, + rioFdTell, + rioFdFlush, NULL, /* update_checksum */ 0, /* current checksum */ 0, /* flags */ @@ -383,24 +365,16 @@ static const rio rioFdsetIO = { { { NULL, 0 } } /* union for io-specific vars */ }; -void rioInitWithConnset(rio *r, connection **conns, int numconns) { - int j; - - *r = rioFdsetIO; - r->io.connset.conns = zmalloc(sizeof(connection *)*numconns); - r->io.connset.state = zmalloc(sizeof(int)*numconns); - memcpy(r->io.connset.conns,conns,sizeof(connection *)*numconns); - for (j = 0; j < numconns; j++) r->io.connset.state[j] = 0; - r->io.connset.numconns = numconns; - r->io.connset.pos = 0; - r->io.connset.buf = sdsempty(); +void rioInitWithFd(rio *r, int fd) { + *r = rioFdIO; + r->io.fd.fd = fd; + r->io.fd.pos = 0; + r->io.fd.buf = sdsempty(); } /* release the rio stream. */ -void rioFreeConnset(rio *r) { - zfree(r->io.connset.conns); - zfree(r->io.connset.state); - sdsfree(r->io.connset.buf); +void rioFreeFd(rio *r) { + sdsfree(r->io.fd.buf); } /* ---------------------------- Generic functions ---------------------------- */ diff --git a/src/rio.h b/src/rio.h index fdde7c20e..9576335e8 100644 --- a/src/rio.h +++ b/src/rio.h @@ -77,7 +77,7 @@ struct _rio { off_t buffered; /* Bytes written since last fsync. */ off_t autosync; /* fsync after 'autosync' bytes written. */ } file; - /* Connection object */ + /* Connection object (used to read from socket) */ struct { connection *conn; /* Connection */ off_t pos; /* pos in buf that was returned */ @@ -85,14 +85,12 @@ struct _rio { size_t read_limit; /* don't allow to buffer/read more than that */ size_t read_so_far; /* amount of data read from the rio (not buffered) */ } conn; - /* Multiple FDs target (used to write to N sockets). */ + /* FD target (used to write to pipe). */ struct { - connection **conns; /* Connections */ - int *state; /* Error state of each fd. 0 (if ok) or errno. */ - int numconns; + int fd; /* File descriptor. */ off_t pos; sds buf; - } connset; + } fd; } io; }; @@ -161,9 +159,9 @@ static inline void rioClearErrors(rio *r) { void rioInitWithFile(rio *r, FILE *fp); void rioInitWithBuffer(rio *r, sds s); void rioInitWithConn(rio *r, connection *conn, size_t read_limit); -void rioInitWithConnset(rio *r, connection **conns, int numconns); +void rioInitWithFd(rio *r, int fd); -void rioFreeConnset(rio *r); +void rioFreeFd(rio *r); void rioFreeConn(rio *r, sds* out_remainingBufferedData); size_t rioWriteBulkCount(rio *r, char prefix, long count); diff --git a/src/server.c b/src/server.c index 930a01723..f05ce3151 100644 --- a/src/server.c +++ b/src/server.c @@ -1746,6 +1746,48 @@ void updateCachedTime(void) { server.daylight_active = tm.tm_isdst; } +void checkChildrenDone(void) { + int statloc; + pid_t pid; + + /* If we have a diskless rdb child (note that we support only one concurrent + * child), we want to avoid collecting it's exit status and acting on it + * as long as we didn't finish to drain the pipe, since then we're at risk + * of starting a new fork and a new pipe before we're done with the previous + * one. */ + if (server.rdb_child_pid != -1 && server.rdb_pipe_conns) + return; + + if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) { + int exitcode = WEXITSTATUS(statloc); + int bysignal = 0; + + if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); + + if (pid == -1) { + serverLog(LL_WARNING,"wait3() returned an error: %s. " + "rdb_child_pid = %d, aof_child_pid = %d", + strerror(errno), + (int) server.rdb_child_pid, + (int) server.aof_child_pid); + } else if (pid == server.rdb_child_pid) { + backgroundSaveDoneHandler(exitcode,bysignal); + if (!bysignal && exitcode == 0) receiveChildInfo(); + } else if (pid == server.aof_child_pid) { + backgroundRewriteDoneHandler(exitcode,bysignal); + if (!bysignal && exitcode == 0) receiveChildInfo(); + } else { + if (!ldbRemoveChild(pid)) { + serverLog(LL_WARNING, + "Warning, detected child with unmatched pid: %ld", + (long)pid); + } + } + updateDictResizePolicy(); + closeChildInfoPipe(); + } +} + /* This is our timer interrupt, called server.hz times per second. * Here is where we do a number of things that need to be done asynchronously. * For instance: @@ -1898,37 +1940,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (server.rdb_child_pid != -1 || server.aof_child_pid != -1 || ldbPendingChildren()) { - int statloc; - pid_t pid; - - if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) { - int exitcode = WEXITSTATUS(statloc); - int bysignal = 0; - - if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc); - - if (pid == -1) { - serverLog(LL_WARNING,"wait3() returned an error: %s. " - "rdb_child_pid = %d, aof_child_pid = %d", - strerror(errno), - (int) server.rdb_child_pid, - (int) server.aof_child_pid); - } else if (pid == server.rdb_child_pid) { - backgroundSaveDoneHandler(exitcode,bysignal); - if (!bysignal && exitcode == 0) receiveChildInfo(); - } else if (pid == server.aof_child_pid) { - backgroundRewriteDoneHandler(exitcode,bysignal); - if (!bysignal && exitcode == 0) receiveChildInfo(); - } else { - if (!ldbRemoveChild(pid)) { - serverLog(LL_WARNING, - "Warning, detected child with unmatched pid: %ld", - (long)pid); - } - } - updateDictResizePolicy(); - closeChildInfoPipe(); - } + checkChildrenDone(); } else { /* If there is not a background saving/rewrite in progress check if * we have to save/rewrite now. */ @@ -2081,6 +2093,11 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Handle writes with pending output buffers. */ handleClientsWithPendingWritesUsingThreads(); + /* TODO: How do i handle write barriers flag */ + tlsProcessPendingData(); + /* If tls already has pending unread data don't sleep at all. */ + aeDontWait(server.el, tlsHasPendingData()); + /* Close clients that need to be closed asynchronous */ freeClientsInAsyncFreeQueue(); @@ -2280,6 +2297,7 @@ void initServerConfig(void) { server.aof_rewrite_incremental_fsync = CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC; server.rdb_save_incremental_fsync = CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC; server.rdb_key_save_delay = CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY; + server.key_load_delay = CONFIG_DEFAULT_KEY_LOAD_DELAY; server.aof_load_truncated = CONFIG_DEFAULT_AOF_LOAD_TRUNCATED; server.aof_use_rdb_preamble = CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE; server.pidfile = NULL; @@ -2813,6 +2831,11 @@ void initServer(void) { server.rdb_child_pid = -1; server.aof_child_pid = -1; server.rdb_child_type = RDB_CHILD_TYPE_NONE; + server.rdb_pipe_conns = NULL; + server.rdb_pipe_numconns = 0; + server.rdb_pipe_numconns_writing = 0; + server.rdb_pipe_buff = NULL; + server.rdb_pipe_bufflen = 0; server.rdb_bgsave_scheduled = 0; server.child_info_pipe[0] = -1; server.child_info_pipe[1] = -1; diff --git a/src/server.h b/src/server.h index 1b41c9ac5..89f219033 100644 --- a/src/server.h +++ b/src/server.h @@ -135,6 +135,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_REPL_DISKLESS_SYNC 0 #define CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY 5 #define CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY 0 +#define CONFIG_DEFAULT_KEY_LOAD_DELAY 0 #define CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA 1 #define CONFIG_DEFAULT_SLAVE_READ_ONLY 1 #define CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY 1 @@ -1236,10 +1237,17 @@ struct redisServer { int rdb_child_type; /* Type of save by active child. */ int lastbgsave_status; /* C_OK or C_ERR */ int stop_writes_on_bgsave_err; /* Don't allow writes if can't BGSAVE */ - int rdb_pipe_write_result_to_parent; /* RDB pipes used to return the state */ - int rdb_pipe_read_result_from_child; /* of each slave in diskless SYNC. */ + int rdb_pipe_write; /* RDB pipes used to transfer the rdb */ + int rdb_pipe_read; /* data to the parent process in diskless repl. */ + connection **rdb_pipe_conns; /* Connections which are currently the */ + int rdb_pipe_numconns; /* target of diskless rdb fork child. */ + int rdb_pipe_numconns_writing; /* Number of rdb conns with pending writes. */ + char *rdb_pipe_buff; /* In diskless replication, this buffer holds data */ + int rdb_pipe_bufflen; /* that was read from the the rdb pipe. */ int rdb_key_save_delay; /* Delay in microseconds between keys while * writing the RDB. (for testings) */ + int key_load_delay; /* Delay in microseconds between keys while + * loading aof or rdb. (for testings) */ /* Pipe and data structures for child -> parent info sharing. */ int child_info_pipe[2]; /* Pipe used to write the child_info_data. */ struct { @@ -1779,6 +1787,8 @@ void clearReplicationId2(void); void chopReplicationBacklog(void); void replicationCacheMasterUsingMyself(void); void feedReplicationBacklog(void *ptr, size_t len); +void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask); +void rdbPipeWriteHandlerConnRemoved(struct connection *conn); /* Generic persistence functions */ void startLoadingFile(FILE* fp, char* filename); @@ -1944,6 +1954,7 @@ unsigned int LRU_CLOCK(void); const char *evictPolicyToString(void); struct redisMemOverhead *getMemoryOverheadData(void); void freeMemoryOverheadData(struct redisMemOverhead *mh); +void checkChildrenDone(void); #define RESTART_SERVER_NONE 0 #define RESTART_SERVER_GRACEFULLY (1<<0) /* Do proper shutdown. */ diff --git a/src/tls.c b/src/tls.c index dabb2ee0f..c939ebf55 100644 --- a/src/tls.c +++ b/src/tls.c @@ -30,6 +30,7 @@ #include "server.h" #include "connhelpers.h" +#include "adlist.h" #ifdef USE_OPENSSL @@ -41,6 +42,10 @@ extern ConnectionType CT_Socket; SSL_CTX *redis_tls_ctx; +/* list of connections with pending data already read from the socket, but not + * served to the reader yet. */ +static list *pending_list = NULL; + void tlsInit(void) { ERR_load_crypto_strings(); SSL_load_error_strings(); @@ -49,6 +54,8 @@ void tlsInit(void) { if (!RAND_poll()) { serverLog(LL_WARNING, "OpenSSL: Failed to seed random number generator."); } + + pending_list = listCreate(); } int tlsConfigureServer(void) { @@ -188,6 +195,7 @@ typedef struct tls_connection { int flags; SSL *ssl; char *ssl_error; + listNode *pending_list_node; } tls_connection; connection *connCreateTLS(void) { @@ -288,11 +296,7 @@ void updateSSLEvent(tls_connection *conn) { aeDeleteFileEvent(server.el, conn->c.fd, AE_WRITABLE); } - -static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) { - UNUSED(el); - UNUSED(fd); - tls_connection *conn = clientData; +static void tlsHandleEvent(tls_connection *conn, int mask) { int ret; TLSCONN_DEBUG("tlsEventHandler(): fd=%d, state=%d, mask=%d, r=%d, w=%d, flags=%d", @@ -369,6 +373,15 @@ static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, in if ((mask & AE_READABLE) && conn->c.read_handler) { if (!callHandler((connection *) conn, conn->c.read_handler)) return; + if (SSL_has_pending(conn->ssl)) { + if (!conn->pending_list_node) { + listAddNodeTail(pending_list, conn); + conn->pending_list_node = listLast(pending_list); + } + } else if (conn->pending_list_node) { + listDelNode(pending_list, conn->pending_list_node); + conn->pending_list_node = NULL; + } } if ((mask & AE_WRITABLE) && conn->c.write_handler) { @@ -382,6 +395,13 @@ static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, in updateSSLEvent(conn); } +static void tlsEventHandler(struct aeEventLoop *el, int fd, void *clientData, int mask) { + UNUSED(el); + UNUSED(fd); + tls_connection *conn = clientData; + tlsHandleEvent(conn, mask); +} + static void connTLSClose(connection *conn_) { tls_connection *conn = (tls_connection *) conn_; @@ -395,6 +415,11 @@ static void connTLSClose(connection *conn_) { conn->ssl_error = NULL; } + if (conn->pending_list_node) { + listDelNode(pending_list, conn->pending_list_node); + conn->pending_list_node = NULL; + } + CT_Socket.close(conn_); } @@ -610,17 +635,6 @@ exit: return nread; } -/* TODO: This is probably not the right thing to do, but as we handle proxying from child - * processes we'll probably not need any shutdown mechanism anyway so this is just a - * place holder for now. - */ -static int connTLSShutdown(connection *conn_, int how) { - UNUSED(how); - tls_connection *conn = (tls_connection *) conn_; - - return SSL_shutdown(conn->ssl); -} - ConnectionType CT_TLS = { .ae_handler = tlsEventHandler, .accept = connTLSAccept, @@ -635,9 +649,25 @@ ConnectionType CT_TLS = { .sync_write = connTLSSyncWrite, .sync_read = connTLSSyncRead, .sync_readline = connTLSSyncReadLine, - .shutdown = connTLSShutdown }; +int tlsHasPendingData() { + if (!pending_list) + return 0; + return listLength(pending_list) > 0; +} + +void tlsProcessPendingData() { + listIter li; + listNode *ln; + + listRewind(pending_list,&li); + while((ln = listNext(&li))) { + tls_connection *conn = listNodeValue(ln); + tlsHandleEvent(conn, AE_READABLE); + } +} + #else /* USE_OPENSSL */ void tlsInit(void) { @@ -666,4 +696,11 @@ connection *connCreateAcceptedTLS(int fd, int require_auth) { return NULL; } +int tlsHasPendingData() { + return 0; +} + +void tlsProcessPendingData() { +} + #endif diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 43bc684b4..192137e87 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -29,9 +29,6 @@ start_server {tags {"repl"}} { $slave slaveof $master_host $master_port test {Slave enters handshake} { - if {$::tls} { - fail "TLS with repl-diskless-sync not supported yet." - } wait_for_condition 50 1000 { [string match *handshake* [$slave role]] } else { @@ -187,10 +184,6 @@ start_server {tags {"repl"}} { } foreach mdl {no yes} { - if {$::tls && $mdl eq "yes"} { - puts "** Skipping test: TLS with repl-diskless-sync not supported yet." - continue - } foreach sdl {disabled swapdb} { start_server {tags {"repl"}} { set master [srv 0 client] @@ -327,9 +320,6 @@ start_server {tags {"repl"}} { } test {slave fails full sync and diskless load swapdb recoveres it} { - if {$::tls} { - fail "" - } start_server {tags {"repl"}} { set slave [srv 0 client] set slave_host [srv 0 host] @@ -397,10 +387,6 @@ test {slave fails full sync and diskless load swapdb recoveres it} { } test {diskless loading short read} { - if {$::tls} { - fail "TLS with repl-diskless-sync not supported yet." - } - start_server {tags {"repl"}} { set replica [srv 0 client] set replica_host [srv 0 host] @@ -480,3 +466,159 @@ test {diskless loading short read} { } } +# get current stime and utime metrics for a thread (since it's creation) +proc get_cpu_metrics { statfile } { + if { [ catch { + set fid [ open $statfile r ] + set data [ read $fid 1024 ] + ::close $fid + set data [ split $data ] + + ;## number of jiffies it has been scheduled... + set utime [ lindex $data 13 ] + set stime [ lindex $data 14 ] + } err ] } { + error "assertion:can't parse /proc: $err" + } + set mstime [clock milliseconds] + return [ list $mstime $utime $stime ] +} + +# compute %utime and %stime of a thread between two measurements +proc compute_cpu_usage {start end} { + set clock_ticks [exec getconf CLK_TCK] + # convert ms time to jiffies and calc delta + set dtime [ expr { ([lindex $end 0] - [lindex $start 0]) * double($clock_ticks) / 1000 } ] + set utime [ expr { [lindex $end 1] - [lindex $start 1] } ] + set stime [ expr { [lindex $end 2] - [lindex $start 2] } ] + set pucpu [ expr { ($utime / $dtime) * 100 } ] + set pscpu [ expr { ($stime / $dtime) * 100 } ] + return [ list $pucpu $pscpu ] +} + + +# test diskless rdb pipe with multiple replicas, which may drop half way +start_server {tags {"repl"}} { + set master [srv 0 client] + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 1 + set master_host [srv 0 host] + set master_port [srv 0 port] + # put enough data in the db that the rdb file will be bigger than the socket buffers + # and since we'll have key-load-delay of 100, 10000 keys will take at least 1 second + # we also need the replica to process requests during transfer (which it does only once in 2mb) + $master debug populate 10000 test 10000 + $master config set rdbcompression no + foreach all_drop {no slow fast all} { + test "diskless $all_drop replicas drop during rdb pipe" { + set replicas {} + set replicas_alive {} + # start one replica that will read the rdb fast, and one that will be slow + start_server {} { + lappend replicas [srv 0 client] + lappend replicas_alive [srv 0 client] + start_server {} { + lappend replicas [srv 0 client] + lappend replicas_alive [srv 0 client] + + # start replication + # it's enough for just one replica to be slow, and have it's write handler enabled + # so that the whole rdb generation process is bound to that + [lindex $replicas 0] config set repl-diskless-load swapdb + [lindex $replicas 0] config set key-load-delay 100 + [lindex $replicas 0] replicaof $master_host $master_port + [lindex $replicas 1] replicaof $master_host $master_port + + # wait for the replicas to start reading the rdb + # using the log file since the replica only responds to INFO once in 2mb + wait_for_log_message -1 "*Loading DB in memory*" 8 800 10 + + set master_statfile [format "/proc/%s/stat" [srv -2 pid]] + set master_start_metrics [get_cpu_metrics $master_statfile] + set start_time [clock seconds] + + # wait a while so that the pipe socket writer will be + # blocked on write (since replica 0 is slow to read from the socket) + after 500 + + # add some command to be present in the command stream after the rdb. + $master incr $all_drop + + # disconnect replicas depending on the current test + if {$all_drop == "all" || $all_drop == "fast"} { + exec kill [srv 0 pid] + set replicas_alive [lreplace $replicas_alive 1 1] + } + if {$all_drop == "all" || $all_drop == "slow"} { + exec kill [srv -1 pid] + set replicas_alive [lreplace $replicas_alive 0 0] + } + + # wait for rdb child to exit + wait_for_condition 500 100 { + [s -2 rdb_bgsave_in_progress] == 0 + } else { + fail "rdb child didn't terminate" + } + + # make sure we got what we were aiming for, by looking for the message in the log file + if {$all_drop == "all"} { + wait_for_log_message -2 "*Diskless rdb transfer, last replica dropped, killing fork child*" 12 1 1 + } + if {$all_drop == "no"} { + wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 2 replicas still up*" 12 1 1 + } + if {$all_drop == "slow" || $all_drop == "fast"} { + wait_for_log_message -2 "*Diskless rdb transfer, done reading from pipe, 1 replicas still up*" 12 1 1 + } + + # make sure we don't have a busy loop going thought epoll_wait + set master_end_metrics [get_cpu_metrics $master_statfile] + set time_elapsed [expr {[clock seconds]-$start_time}] + set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] + set master_utime [lindex $master_cpu 0] + set master_stime [lindex $master_cpu 1] + if {$::verbose} { + puts "elapsed: $time_elapsed" + puts "master utime: $master_utime" + puts "master stime: $master_stime" + } + if {$all_drop == "all" || $all_drop == "slow"} { + assert {$master_utime < 30} + assert {$master_stime < 30} + } + if {$all_drop == "none" || $all_drop == "fast"} { + assert {$master_utime < 15} + assert {$master_stime < 15} + } + + # verify the data integrity + foreach replica $replicas_alive { + # Wait that replicas acknowledge they are online so + # we are sure that DBSIZE and DEBUG DIGEST will not + # fail because of timing issues. + wait_for_condition 50 100 { + [lindex [$replica role] 3] eq {connected} + } else { + fail "replicas still not connected after some time" + } + + # Make sure that replicas and master have same + # number of keys + wait_for_condition 50 100 { + [$master dbsize] == [$replica dbsize] + } else { + fail "Different number of keys between master and replicas after too long time." + } + + # Check digests + set digest [$master debug digest] + set digest0 [$replica debug digest] + assert {$digest ne 0000000000000000000000000000000000000000} + assert {$digest eq $digest0} + } + } + } + } + } +} From 8704be694793033b868cca0366e8e97ca8894dec Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 19 Aug 2019 12:18:25 +0300 Subject: [PATCH 169/320] TLS: Implement support for write barrier. --- TLS.md | 10 +++----- src/aof.c | 4 +++ src/cluster.c | 2 +- src/connection.c | 32 +++++++++++++++++++++--- src/connection.h | 14 +++++++++-- src/debug.c | 6 +++++ src/networking.c | 9 +++---- src/server.c | 11 +++++---- src/server.h | 1 + src/tls.c | 51 +++++++++++++++++++++++++++++++-------- tests/integration/aof.tcl | 31 ++++++++++++++++++++++++ 11 files changed, 138 insertions(+), 33 deletions(-) diff --git a/TLS.md b/TLS.md index 90ed08e7c..c627f814c 100644 --- a/TLS.md +++ b/TLS.md @@ -57,22 +57,18 @@ Connections Connection abstraction API is mostly done and seems to hold well for hiding implementation details between TLS and TCP. -1. Still need to implement the equivalent of AE_BARRIER. Because TLS - socket-level read/write events don't correspond to logical operations, this - should probably be done at the Read/Write handler level. - -2. Multi-threading I/O is not supported. The main issue to address is the need +1. Multi-threading I/O is not supported. The main issue to address is the need to manipulate AE based on OpenSSL return codes. We can either propagate this out of the thread, or explore ways of further optimizing MT I/O by having event loops that live inside the thread and borrow connections in/out. -3. Finish cleaning up the implementation. Make sure all error cases are handled +2. Finish cleaning up the implementation. Make sure all error cases are handled and reflected into connection state, connection state validated before certain operations, etc. - Clean (non-errno) interface to report would-block. - Consistent error reporting. -4. Sync IO for TLS is currently implemented in a hackish way, i.e. making the +3. Sync IO for TLS is currently implemented in a hackish way, i.e. making the socket blocking and configuring socket-level timeout. This means the timeout value may not be so accurate, and there would be a lot of syscall overhead. However I believe that getting rid of syncio completely in favor of pure diff --git a/src/aof.c b/src/aof.c index ef34097ed..181995bbc 100644 --- a/src/aof.c +++ b/src/aof.c @@ -385,6 +385,10 @@ void flushAppendOnlyFile(int force) { * there is much to do about the whole server stopping for power problems * or alike */ + if (server.aof_flush_sleep && sdslen(server.aof_buf)) { + usleep(server.aof_flush_sleep); + } + latencyStartMonitor(latency); nwritten = aofWrite(server.aof_fd,server.aof_buf,sdslen(server.aof_buf)); latencyEndMonitor(latency); diff --git a/src/cluster.c b/src/cluster.c index 639ab1ea7..1d922e87a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -2277,7 +2277,7 @@ void clusterReadHandler(connection *conn) { * from event handlers that will do stuff with the same link later. */ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { if (sdslen(link->sndbuf) == 0 && msglen != 0) - connSetWriteHandler(link->conn, clusterWriteHandler); /* TODO: Handle AE_BARRIER in conns */ + connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1); link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); diff --git a/src/connection.c b/src/connection.c index 601f175bc..85bf572ad 100644 --- a/src/connection.c +++ b/src/connection.c @@ -194,10 +194,14 @@ static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_hand /* Register a write handler, to be called when the connection is writable. * If NULL, the existing handler is removed. */ -static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { +static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier) { if (func == conn->write_handler) return C_OK; conn->write_handler = func; + if (barrier) + conn->flags |= CONN_FLAG_WRITE_BARRIER; + else + conn->flags &= ~CONN_FLAG_WRITE_BARRIER; if (!conn->write_handler) aeDeleteFileEvent(server.el,conn->fd,AE_WRITABLE); else @@ -247,13 +251,35 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD conn->conn_handler = NULL; } + /* Normally we execute the readable event first, and the writable + * event laster. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if WRITE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsynching a file to disk, + * before replying to a client. */ + int invert = conn->flags & CONN_FLAG_WRITE_BARRIER; + + int call_write = (mask & AE_WRITABLE) && conn->write_handler; + int call_read = (mask & AE_READABLE) && conn->read_handler; + /* Handle normal I/O flows */ - if ((mask & AE_READABLE) && conn->read_handler) { + if (!invert && call_read) { if (!callHandler(conn, conn->read_handler)) return; } - if ((mask & AE_WRITABLE) && conn->write_handler) { + /* Fire the writable event. */ + if (call_write) { if (!callHandler(conn, conn->write_handler)) return; } + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && call_read) { + if (!callHandler(conn, conn->read_handler)) return; + } } static int connSocketBlockingConnect(connection *conn, const char *addr, int port, long long timeout) { diff --git a/src/connection.h b/src/connection.h index 3fdcddebe..97622f8d6 100644 --- a/src/connection.h +++ b/src/connection.h @@ -47,6 +47,7 @@ typedef enum { #define CONN_FLAG_IN_HANDLER (1<<0) /* A handler execution is in progress */ #define CONN_FLAG_CLOSE_SCHEDULED (1<<1) /* Closed scheduled by a handler */ +#define CONN_FLAG_WRITE_BARRIER (1<<2) /* Write barrier requested */ typedef void (*ConnectionCallbackFunc)(struct connection *conn); @@ -57,7 +58,7 @@ typedef struct ConnectionType { int (*read)(struct connection *conn, void *buf, size_t buf_len); void (*close)(struct connection *conn); int (*accept)(struct connection *conn, ConnectionCallbackFunc accept_handler); - int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler); + int (*set_write_handler)(struct connection *conn, ConnectionCallbackFunc handler, int barrier); int (*set_read_handler)(struct connection *conn, ConnectionCallbackFunc handler); const char *(*get_last_error)(struct connection *conn); int (*blocking_connect)(struct connection *conn, const char *addr, int port, long long timeout); @@ -144,7 +145,7 @@ static inline int connRead(connection *conn, void *buf, size_t buf_len) { * If NULL, the existing handler is removed. */ static inline int connSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { - return conn->type->set_write_handler(conn, func); + return conn->type->set_write_handler(conn, func, 0); } /* Register a read handler, to be called when the connection is readable. @@ -154,6 +155,15 @@ static inline int connSetReadHandler(connection *conn, ConnectionCallbackFunc fu return conn->type->set_read_handler(conn, func); } +/* Set a write handler, and possibly enable a write barrier, this flag is + * cleared when write handler is changed or removed. + * With barroer enabled, we never fire the event if the read handler already + * fired in the same event loop iteration. Useful when you want to persist + * things to disk before sending replies, and want to do that in a group fashion. */ +static inline int connSetWriteHandlerWithBarrier(connection *conn, ConnectionCallbackFunc func, int barrier) { + return conn->type->set_write_handler(conn, func, barrier); +} + static inline void connClose(connection *conn) { conn->type->close(conn); } diff --git a/src/debug.c b/src/debug.c index a2d61d8ab..c1546c4c1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -319,6 +319,7 @@ void debugCommand(client *c) { "SDSLEN -- Show low level SDS string info representing key and value.", "SEGFAULT -- Crash the server with sigsegv.", "SET-ACTIVE-EXPIRE <0|1> -- Setting it to 0 disables expiring keys in background when they are not accessed (otherwise the Redis behavior). Setting it to 1 reenables back the default.", +"AOF-FLUSH-SLEEP -- Server will sleep before flushing the AOF, this is used for testing", "SLEEP -- Stop the server for . Decimals allowed.", "STRUCTSIZE -- Return the size of different Redis core C structures.", "ZIPLIST -- Show low level info about the ziplist encoding.", @@ -595,6 +596,11 @@ NULL { server.active_expire_enabled = atoi(c->argv[2]->ptr); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"aof-flush-sleep") && + c->argc == 3) + { + server.aof_flush_sleep = atoi(c->argv[2]->ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"lua-always-replicate-commands") && c->argc == 3) { diff --git a/src/networking.c b/src/networking.c index d52eb1eba..97035c931 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1305,19 +1305,18 @@ int handleClientsWithPendingWrites(void) { /* If after the synchronous writes above we still have data to * output to the client, we need to install the writable handler. */ if (clientHasPendingReplies(c)) { - int ae_flags = AE_WRITABLE; + int ae_barrier = 0; /* For the fsync=always policy, we want that a given FD is never * served for reading and writing in the same event loop iteration, * so that in the middle of receiving the query, and serving it * to the client, we'll call beforeSleep() that will do the - * actual fsync of AOF to disk. AE_BARRIER ensures that. */ + * actual fsync of AOF to disk. the write barrier ensures that. */ if (server.aof_state == AOF_ON && server.aof_fsync == AOF_FSYNC_ALWAYS) { - ae_flags |= AE_BARRIER; + ae_barrier = 1; } - /* TODO: Handle write barriers in connection (also see tlsProcessPendingData) */ - if (connSetWriteHandler(c->conn, sendReplyToClient) == C_ERR) { + if (connSetWriteHandlerWithBarrier(c->conn, sendReplyToClient, ae_barrier) == C_ERR) { freeClientAsync(c); } } diff --git a/src/server.c b/src/server.c index f05ce3151..0f5837bc7 100644 --- a/src/server.c +++ b/src/server.c @@ -2048,6 +2048,11 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { void beforeSleep(struct aeEventLoop *eventLoop) { UNUSED(eventLoop); + /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ + tlsProcessPendingData(); + /* If tls still has pending unread data don't sleep at all. */ + aeDontWait(server.el, tlsHasPendingData()); + /* Call the Redis Cluster before sleep function. Note that this function * may change the state of Redis Cluster (from ok to fail or vice versa), * so it's a good idea to call it before serving the unblocked clients @@ -2093,11 +2098,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Handle writes with pending output buffers. */ handleClientsWithPendingWritesUsingThreads(); - /* TODO: How do i handle write barriers flag */ - tlsProcessPendingData(); - /* If tls already has pending unread data don't sleep at all. */ - aeDontWait(server.el, tlsHasPendingData()); - /* Close clients that need to be closed asynchronous */ freeClientsInAsyncFreeQueue(); @@ -2286,6 +2286,7 @@ void initServerConfig(void) { server.aof_rewrite_min_size = AOF_REWRITE_MIN_SIZE; server.aof_rewrite_base_size = 0; server.aof_rewrite_scheduled = 0; + server.aof_flush_sleep = 0; server.aof_last_fsync = time(NULL); server.aof_rewrite_time_last = -1; server.aof_rewrite_time_start = -1; diff --git a/src/server.h b/src/server.h index 89f219033..df5e42c41 100644 --- a/src/server.h +++ b/src/server.h @@ -1192,6 +1192,7 @@ struct redisServer { off_t aof_rewrite_base_size; /* AOF size on latest startup or rewrite. */ off_t aof_current_size; /* AOF current size. */ off_t aof_fsync_offset; /* AOF offset which is already synced to disk. */ + int aof_flush_sleep; /* Micros to sleep before flush. (used by tests) */ int aof_rewrite_scheduled; /* Rewrite once BGSAVE terminates. */ pid_t aof_child_pid; /* PID if rewriting process */ list *aof_rewrite_buf_blocks; /* Hold changes during an AOF rewrite. */ diff --git a/src/tls.c b/src/tls.c index c939ebf55..68d15ef9e 100644 --- a/src/tls.c +++ b/src/tls.c @@ -361,18 +361,47 @@ static void tlsHandleEvent(tls_connection *conn, int mask) { conn->c.conn_handler = NULL; break; case CONN_STATE_CONNECTED: - if ((mask & AE_READABLE) && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)) { - conn->flags &= ~TLS_CONN_FLAG_WRITE_WANT_READ; - if (!callHandler((connection *) conn, conn->c.write_handler)) return; - } + { + int call_read = ((mask & AE_READABLE) && conn->c.read_handler) || + ((mask & AE_WRITABLE) && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)); + int call_write = ((mask & AE_WRITABLE) && conn->c.write_handler) || + ((mask & AE_READABLE) && (conn->flags & TLS_CONN_FLAG_WRITE_WANT_READ)); - if ((mask & AE_WRITABLE) && (conn->flags & TLS_CONN_FLAG_READ_WANT_WRITE)) { + /* Normally we execute the readable event first, and the writable + * event laster. This is useful as sometimes we may be able + * to serve the reply of a query immediately after processing the + * query. + * + * However if WRITE_BARRIER is set in the mask, our application is + * asking us to do the reverse: never fire the writable event + * after the readable. In such a case, we invert the calls. + * This is useful when, for instance, we want to do things + * in the beforeSleep() hook, like fsynching a file to disk, + * before replying to a client. */ + int invert = conn->c.flags & CONN_FLAG_WRITE_BARRIER; + + if (!invert && call_read) { conn->flags &= ~TLS_CONN_FLAG_READ_WANT_WRITE; if (!callHandler((connection *) conn, conn->c.read_handler)) return; } - if ((mask & AE_READABLE) && conn->c.read_handler) { + /* Fire the writable event. */ + if (call_write) { + conn->flags &= ~TLS_CONN_FLAG_WRITE_WANT_READ; + if (!callHandler((connection *) conn, conn->c.write_handler)) return; + } + + /* If we have to invert the call, fire the readable event now + * after the writable one. */ + if (invert && call_read) { + conn->flags &= ~TLS_CONN_FLAG_READ_WANT_WRITE; if (!callHandler((connection *) conn, conn->c.read_handler)) return; + } + + /* If SSL has pending that, already read from the socket, we're at + * risk of not calling the read handler again, make sure to add it + * to a list of pending connection that should be handled anyway. */ + if ((mask & AE_READABLE)) { if (SSL_has_pending(conn->ssl)) { if (!conn->pending_list_node) { listAddNodeTail(pending_list, conn); @@ -384,10 +413,8 @@ static void tlsHandleEvent(tls_connection *conn, int mask) { } } - if ((mask & AE_WRITABLE) && conn->c.write_handler) { - if (!callHandler((connection *) conn, conn->c.write_handler)) return; - } break; + } default: break; } @@ -535,8 +562,12 @@ static const char *connTLSGetLastError(connection *conn_) { return NULL; } -int connTLSSetWriteHandler(connection *conn, ConnectionCallbackFunc func) { +int connTLSSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier) { conn->write_handler = func; + if (barrier) + conn->flags |= CONN_FLAG_WRITE_BARRIER; + else + conn->flags &= ~CONN_FLAG_WRITE_BARRIER; updateSSLEvent((tls_connection *) conn); return C_OK; } diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index e276a6254..2734de7f1 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -257,4 +257,35 @@ tags {"aof"} { r expire x -1 } } + + start_server {overrides {appendonly {yes} appendfilename {appendonly.aof} appendfsync always}} { + test {AOF fsync always barrier issue} { + set rd [redis_deferring_client] + # Set a sleep when aof is flushed, so that we have a chance to look + # at the aof size and detect if the response of an incr command + # arrives before the data was written (and hopefully fsynced) + # We create a big reply, which will hopefully not have room in the + # socket buffers, and will install a write handler, then we sleep + # a big and issue the incr command, hoping that the last portion of + # the output buffer write, and the processing of the incr will happen + # in the same event loop cycle. + # Since the socket buffers and timing are unpredictable, we fuzz this + # test with slightly different sizes and sleeps a few times. + for {set i 0} {$i < 10} {incr i} { + r debug aof-flush-sleep 0 + r del x + r setrange x [expr {int(rand()*5000000)+10000000}] x + r debug aof-flush-sleep 500000 + set aof [file join [lindex [r config get dir] 1] appendonly.aof] + set size1 [file size $aof] + $rd get x + after [expr {int(rand()*30)}] + $rd incr new_value + $rd read + $rd read + set size2 [file size $aof] + assert {$size1 != $size2} + } + } + } } From df08b624bdc781831d59e691fbb6c3738edec620 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Thu, 12 Sep 2019 11:10:22 +0300 Subject: [PATCH 170/320] TLS: Configuration options. Add configuration options for TLS protocol versions, ciphers/cipher suites selection, etc. --- TLS.md | 54 ++++------ redis.conf | 24 +++++ src/Makefile | 2 + src/config.c | 170 +++++++++++++++++++++--------- src/replication.c | 11 +- src/sentinel.c | 5 +- src/server.c | 9 +- src/server.h | 24 +++-- src/tls.c | 137 ++++++++++++++++++------ tests/integration/replication.tcl | 48 +++++---- tests/sentinel/run.tcl | 1 + tests/support/redis.tcl | 5 +- tests/unit/tls.tcl | 84 ++++++++++++++- 13 files changed, 414 insertions(+), 160 deletions(-) diff --git a/TLS.md b/TLS.md index c627f814c..76fe0be2e 100644 --- a/TLS.md +++ b/TLS.md @@ -48,45 +48,35 @@ both TCP and TLS available, but you'll need to assign different ports. To make a Replica connect to the master using TLS, use `--tls-replication yes`, and to make Redis Cluster use TLS across nodes use `--tls-cluster yes`. -**NOTE: This is still very much work in progress and some configuration is still -missing or may change.** - Connections ----------- -Connection abstraction API is mostly done and seems to hold well for hiding -implementation details between TLS and TCP. +All socket operations now go through a connection abstraction layer that hides +I/O and read/write event handling from the caller. -1. Multi-threading I/O is not supported. The main issue to address is the need - to manipulate AE based on OpenSSL return codes. We can either propagate this - out of the thread, or explore ways of further optimizing MT I/O by having - event loops that live inside the thread and borrow connections in/out. +**Multi-threading I/O is not currently supported for TLS**, as a TLS connection +needs to do its own manipulation of AE events which is not thread safe. The +solution is probably to manage independent AE loops for I/O threads and longer +term association of connections with threads. This may potentially improve +overall performance as well. -2. Finish cleaning up the implementation. Make sure all error cases are handled - and reflected into connection state, connection state validated before - certain operations, etc. - - Clean (non-errno) interface to report would-block. - - Consistent error reporting. +Sync IO for TLS is currently implemented in a hackish way, i.e. making the +socket blocking and configuring socket-level timeout. This means the timeout +value may not be so accurate, and there would be a lot of syscall overhead. +However I believe that getting rid of syncio completely in favor of pure async +work is probably a better move than trying to fix that. For replication it would +probably not be so hard. For cluster keys migration it might be more difficult, +but there are probably other good reasons to improve that part anyway. -3. Sync IO for TLS is currently implemented in a hackish way, i.e. making the - socket blocking and configuring socket-level timeout. This means the timeout - value may not be so accurate, and there would be a lot of syscall overhead. - However I believe that getting rid of syncio completely in favor of pure - async work is probably a better move than trying to fix that. For replication - it would probably not be so hard. For cluster keys migration it might be more - difficult, but there are probably other good reasons to improve that part - anyway. +To-Do List +========== -TLS Features ------------- - -1. Add metrics to INFO. -2. Add certificate authentication configuration (i.e. option to skip client -auth, master auth, etc.). -3. Add TLS cipher configuration options. -4. [Optional] Add session caching support. Check if/how it's handled by clients - to assess how useful/important it is. +Additional TLS Features +----------------------- +1. Add metrics to INFO? +2. Add session caching support. Check if/how it's handled by clients to assess + how useful/important it is. redis-benchmark --------------- @@ -100,8 +90,8 @@ probably to migrate to hiredis async mode. redis-cli --------- -1. Support tls in --slave and --rdb +1. Add support for TLS in --slave and --rdb modes. Others ------ diff --git a/redis.conf b/redis.conf index 2af422a93..1f966badb 100644 --- a/redis.conf +++ b/redis.conf @@ -173,6 +173,30 @@ tcp-keepalive 300 # # tls-cluster yes +# Explicitly specify TLS versions to support. Allowed values are case insensitive +# and include "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" (OpenSSL >= 1.1.1) or +# "default" which is currently >= TLSv1.1. +# +# tls-protocols TLSv1.2 + +# Configure allowed ciphers. See the ciphers(1ssl) manpage for more information +# about the syntax of this string. +# +# Note: this configuration applies only to <= TLSv1.2. +# +# tls-ciphers DEFAULT:!MEDIUM + +# Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more +# information about the syntax of this string, and specifically for TLSv1.3 +# ciphersuites. +# +# tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + +# When choosing a cipher, use the server's preference instead of the client +# preference. By default, the server follows the client's preference. +# +# tls-prefer-server-cipher yes + ################################# GENERAL ##################################### # By default Redis does not run as a daemon. Use 'yes' if you need it. diff --git a/src/Makefile b/src/Makefile index b43b743dc..b020f17cf 100644 --- a/src/Makefile +++ b/src/Makefile @@ -93,6 +93,8 @@ else ifeq ($(uname_S),Darwin) # Darwin FINAL_LIBS+= -ldl + OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include + OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib else ifeq ($(uname_S),AIX) # AIX diff --git a/src/config.c b/src/config.c index 792e45057..bb04cb45b 100644 --- a/src/config.c +++ b/src/config.c @@ -219,7 +219,7 @@ void queueLoadModule(sds path, sds *argv, int argc) { } void loadServerConfigFromString(char *config) { - char *err = NULL; + const char *err = NULL; int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; @@ -286,15 +286,6 @@ void loadServerConfigFromString(char *config) { if (server.port < 0 || server.port > 65535) { err = "Invalid port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"tls-port") && argc == 2) { -#ifdef USE_OPENSSL - server.tls_port = atoi(argv[1]); - if (server.port < 0 || server.port > 65535) { - err = "Invalid port"; goto loaderr; - } -#else - err = "TLS not supported"; goto loaderr; -#endif } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { server.tcp_backlog = atoi(argv[1]); if (server.tcp_backlog < 0) { @@ -806,24 +797,42 @@ void loadServerConfigFromString(char *config) { err = sentinelHandleConfiguration(argv+1,argc-1); if (err) goto loaderr; } - } else if (!strcasecmp(argv[0],"tls-cert-file") && argc == 2) { - zfree(server.tls_cert_file); - server.tls_cert_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-key-file") && argc == 2) { - zfree(server.tls_key_file); - server.tls_key_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-dh-params-file") && argc == 2) { - zfree(server.tls_dh_params_file); - server.tls_dh_params_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-ca-cert-file") && argc == 2) { - zfree(server.tls_ca_cert_file); - server.tls_ca_cert_file = zstrdup(argv[1]); +#ifdef USE_OPENSSL + } else if (!strcasecmp(argv[0],"tls-port") && argc == 2) { + server.tls_port = atoi(argv[1]); + if (server.port < 0 || server.port > 65535) { + err = "Invalid tls-port"; goto loaderr; + } } else if (!strcasecmp(argv[0],"tls-cluster") && argc == 2) { server.tls_cluster = yesnotoi(argv[1]); } else if (!strcasecmp(argv[0],"tls-replication") && argc == 2) { server.tls_replication = yesnotoi(argv[1]); } else if (!strcasecmp(argv[0],"tls-auth-clients") && argc == 2) { server.tls_auth_clients = yesnotoi(argv[1]); + } else if (!strcasecmp(argv[0],"tls-cert-file") && argc == 2) { + zfree(server.tls_ctx_config.cert_file); + server.tls_ctx_config.cert_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-key-file") && argc == 2) { + zfree(server.tls_ctx_config.key_file); + server.tls_ctx_config.key_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-dh-params-file") && argc == 2) { + zfree(server.tls_ctx_config.dh_params_file); + server.tls_ctx_config.dh_params_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-ca-cert-file") && argc == 2) { + zfree(server.tls_ctx_config.ca_cert_file); + server.tls_ctx_config.ca_cert_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-protocols") && argc >= 2) { + zfree(server.tls_ctx_config.protocols); + server.tls_ctx_config.protocols = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-ciphers") && argc == 2) { + zfree(server.tls_ctx_config.ciphers); + server.tls_ctx_config.ciphers = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-ciphersuites") && argc == 2) { + zfree(server.tls_ctx_config.ciphersuites); + server.tls_ctx_config.ciphersuites = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-prefer-server-ciphers") && argc == 2) { + server.tls_ctx_config.prefer_server_ciphers = yesnotoi(argv[1]); +#endif /* USE_OPENSSL */ } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -1268,46 +1277,90 @@ void configSetCommand(client *c) { "appendfsync",server.aof_fsync,aof_fsync_enum) { } config_set_enum_field( "repl-diskless-load",server.repl_diskless_load,repl_diskless_load_enum) { - +#ifdef USE_OPENSSL /* TLS fields. */ } config_set_special_field("tls-cert-file") { - if (tlsConfigure((char *) o->ptr, server.tls_key_file, - server.tls_dh_params_file, server.tls_ca_cert_file) == C_ERR) { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.cert_file = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { addReplyError(c, "Unable to configure tls-cert-file. Check server logs."); return; } - zfree(server.tls_cert_file); - server.tls_cert_file = zstrdup(o->ptr); + zfree(server.tls_ctx_config.cert_file); + server.tls_ctx_config.cert_file = zstrdup(o->ptr); } config_set_special_field("tls-key-file") { - if (tlsConfigure(server.tls_cert_file, (char *) o->ptr, - server.tls_dh_params_file, server.tls_ca_cert_file) == C_ERR) { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.key_file = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { addReplyError(c, "Unable to configure tls-key-file. Check server logs."); return; } - zfree(server.tls_key_file); - server.tls_key_file = zstrdup(o->ptr); + zfree(server.tls_ctx_config.key_file); + server.tls_ctx_config.key_file = zstrdup(o->ptr); } config_set_special_field("tls-dh-params-file") { - if (tlsConfigure(server.tls_cert_file, server.tls_key_file, - (char *) o->ptr, server.tls_ca_cert_file) == C_ERR) { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.dh_params_file = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { addReplyError(c, "Unable to configure tls-dh-params-file. Check server logs."); return; } - zfree(server.tls_dh_params_file); - server.tls_dh_params_file = zstrdup(o->ptr); + zfree(server.tls_ctx_config.dh_params_file); + server.tls_ctx_config.dh_params_file = zstrdup(o->ptr); } config_set_special_field("tls-ca-cert-file") { - if (tlsConfigure(server.tls_cert_file, server.tls_key_file, - server.tls_dh_params_file, (char *) o->ptr) == C_ERR) { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.ca_cert_file = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { addReplyError(c, "Unable to configure tls-ca-cert-file. Check server logs."); return; } - zfree(server.tls_ca_cert_file); - server.tls_ca_cert_file = zstrdup(o->ptr); + zfree(server.tls_ctx_config.ca_cert_file); + server.tls_ctx_config.ca_cert_file = zstrdup(o->ptr); } config_set_bool_field("tls-auth-clients", server.tls_auth_clients) { - + } config_set_bool_field("tls-replication", server.tls_replication) { + } config_set_bool_field("tls-cluster", server.tls_cluster) { + } config_set_special_field("tls-protocols") { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.protocols = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { + addReplyError(c, + "Unable to configure tls-protocols. Check server logs."); + return; + } + zfree(server.tls_ctx_config.protocols); + server.tls_ctx_config.protocols = zstrdup(o->ptr); + } config_set_special_field("tls-ciphers") { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.ciphers = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { + addReplyError(c, + "Unable to configure tls-ciphers. Check server logs."); + return; + } + zfree(server.tls_ctx_config.ciphers); + server.tls_ctx_config.ciphers = zstrdup(o->ptr); + } config_set_special_field("tls-ciphersuites") { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.ciphersuites = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { + addReplyError(c, + "Unable to configure tls-ciphersuites. Check server logs."); + return; + } + zfree(server.tls_ctx_config.ciphersuites); + server.tls_ctx_config.ciphersuites = zstrdup(o->ptr); + } config_set_special_field("tls-prefer-server-ciphers") { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.prefer_server_ciphers = yesnotoi(o->ptr); + if (tlsConfigure(&tmpctx) == C_ERR) { + addReplyError(c, "Unable to reconfigure TLS. Check server logs."); + return; + } + server.tls_ctx_config.prefer_server_ciphers = tmpctx.prefer_server_ciphers; +#endif /* USE_OPENSSL */ /* Everyhing else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -1381,10 +1434,15 @@ void configGetCommand(client *c) { config_get_string_field("pidfile",server.pidfile); config_get_string_field("slave-announce-ip",server.slave_announce_ip); config_get_string_field("replica-announce-ip",server.slave_announce_ip); - config_get_string_field("tls-cert-file",server.tls_cert_file); - config_get_string_field("tls-key-file",server.tls_key_file); - config_get_string_field("tls-dh-params-file",server.tls_dh_params_file); - config_get_string_field("tls-ca-cert-file",server.tls_ca_cert_file); +#ifdef USE_OPENSSL + config_get_string_field("tls-cert-file",server.tls_ctx_config.cert_file); + config_get_string_field("tls-key-file",server.tls_ctx_config.key_file); + config_get_string_field("tls-dh-params-file",server.tls_ctx_config.dh_params_file); + config_get_string_field("tls-ca-cert-file",server.tls_ctx_config.ca_cert_file); + config_get_string_field("tls-protocols",server.tls_ctx_config.protocols); + config_get_string_field("tls-ciphers",server.tls_ctx_config.ciphers); + config_get_string_field("tls-ciphersuites",server.tls_ctx_config.ciphersuites); +#endif /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); @@ -1476,7 +1534,8 @@ void configGetCommand(client *c) { config_get_bool_field("tls-cluster",server.tls_cluster); config_get_bool_field("tls-replication",server.tls_replication); config_get_bool_field("tls-auth-clients",server.tls_auth_clients); - + config_get_bool_field("tls-prefer-server-ciphers", + server.tls_ctx_config.prefer_server_ciphers); /* Enum values */ config_get_enum_field("maxmemory-policy", server.maxmemory_policy,maxmemory_policy_enum); @@ -1590,6 +1649,7 @@ void configGetCommand(client *c) { } matches++; } + setDeferredMapLen(c,replylen,matches); } @@ -2200,9 +2260,6 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); - rewriteConfigYesNoOption(state,"tls-cluster",server.tls_cluster,0); - rewriteConfigYesNoOption(state,"tls-replication",server.tls_replication,0); - rewriteConfigYesNoOption(state,"tls-auth-clients",server.tls_auth_clients,1); rewriteConfigBindOption(state); rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); @@ -2282,10 +2339,19 @@ int rewriteConfig(char *path) { rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); rewriteConfigNumericalOption(state,"rdb-key-save-delay",server.rdb_key_save_delay,CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY); rewriteConfigNumericalOption(state,"key-load-delay",server.key_load_delay,CONFIG_DEFAULT_KEY_LOAD_DELAY); - rewriteConfigStringOption(state,"tls-cert-file",server.tls_cert_file,NULL); - rewriteConfigStringOption(state,"tls-key-file",server.tls_key_file,NULL); - rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_dh_params_file,NULL); - rewriteConfigStringOption(state,"tls-ca-cert-file",server.tls_ca_cert_file,NULL); +#ifdef USE_OPENSSL + rewriteConfigYesNoOption(state,"tls-cluster",server.tls_cluster,0); + rewriteConfigYesNoOption(state,"tls-replication",server.tls_replication,0); + rewriteConfigYesNoOption(state,"tls-auth-clients",server.tls_auth_clients,1); + rewriteConfigStringOption(state,"tls-cert-file",server.tls_ctx_config.cert_file,NULL); + rewriteConfigStringOption(state,"tls-key-file",server.tls_ctx_config.key_file,NULL); + rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_ctx_config.dh_params_file,NULL); + rewriteConfigStringOption(state,"tls-ca-cert-file",server.tls_ctx_config.ca_cert_file,NULL); + rewriteConfigStringOption(state,"tls-protocols",server.tls_ctx_config.protocols,NULL); + rewriteConfigStringOption(state,"tls-ciphers",server.tls_ctx_config.ciphers,NULL); + rewriteConfigStringOption(state,"tls-ciphersuites",server.tls_ctx_config.ciphersuites,NULL); + rewriteConfigYesNoOption(state,"tls-prefer-server-ciphers",server.tls_ctx_config.prefer_server_ciphers,0); +#endif /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); diff --git a/src/replication.c b/src/replication.c index 9f5c19053..98a7dd9e6 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2023,11 +2023,14 @@ void syncWithMaster(connection *conn) { /* Set the slave port, so that Master's INFO command can list the * slave listening port correctly. */ if (server.repl_state == REPL_STATE_SEND_PORT) { - sds port = sdsfromlonglong(server.slave_announce_port ? - server.slave_announce_port : server.port); + int port; + if (server.slave_announce_port) port = server.slave_announce_port; + else if (server.tls_replication && server.tls_port) port = server.tls_port; + else port = server.port; + sds portstr = sdsfromlonglong(port); err = sendSynchronousCommand(SYNC_CMD_WRITE,conn,"REPLCONF", - "listening-port",port, NULL); - sdsfree(port); + "listening-port",portstr, NULL); + sdsfree(portstr); if (err) goto write_error; sdsfree(err); server.repl_state = REPL_STATE_RECEIVE_PORT; diff --git a/src/sentinel.c b/src/sentinel.c index 95f189485..92869805d 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2612,8 +2612,9 @@ int sentinelSendHello(sentinelRedisInstance *ri) { return C_ERR; announce_ip = ip; } - announce_port = sentinel.announce_port ? - sentinel.announce_port : server.port; + if (sentinel.announce_port) announce_port = sentinel.announce_port; + else if (server.tls_replication && server.tls_port) announce_port = server.tls_port; + else announce_port = server.port; /* Format and send the Hello message. */ snprintf(payload,sizeof(payload), diff --git a/src/server.c b/src/server.c index 0f5837bc7..3de1347cb 100644 --- a/src/server.c +++ b/src/server.c @@ -2451,9 +2451,6 @@ void initServerConfig(void) { * script to the slave / AOF. This is the new way starting from * Redis 5. However it is possible to revert it via redis.conf. */ server.lua_always_replicate_commands = 1; - - /* TLS */ - server.tls_auth_clients = 1; } extern char **environ; @@ -2770,7 +2767,7 @@ void initServer(void) { server.clients_paused = 0; server.system_memory_size = zmalloc_get_memory_size(); - if (server.tls_port && tlsConfigureServer() == C_ERR) { + if (server.tls_port && tlsConfigure(&server.tls_ctx_config) == C_ERR) { serverLog(LL_WARNING, "Failed to configure TLS. Check logs for more info."); exit(1); } @@ -3943,7 +3940,7 @@ sds genRedisInfoString(char *section) { #endif (long) getpid(), server.runid, - server.port, + server.port ? server.port : server.tls_port, (intmax_t)uptime, (intmax_t)(uptime/(3600*24)), server.hz, @@ -4554,7 +4551,7 @@ void redisAsciiArt(void) { if (!show_logo) { serverLog(LL_NOTICE, "Running mode=%s, port=%d.", - mode, server.port + mode, server.port ? server.port : server.tls_port ); } else { snprintf(buf,1024*16,ascii_logo, diff --git a/src/server.h b/src/server.h index df5e42c41..5c7849732 100644 --- a/src/server.h +++ b/src/server.h @@ -1029,6 +1029,21 @@ struct malloc_stats { size_t allocator_resident; }; +/*----------------------------------------------------------------------------- + * TLS Context Configuration + *----------------------------------------------------------------------------*/ + +typedef struct redisTLSContextConfig { + char *cert_file; + char *key_file; + char *dh_params_file; + char *ca_cert_file; + char *protocols; + char *ciphers; + char *ciphersuites; + int prefer_server_ciphers; +} redisTLSContextConfig; + /*----------------------------------------------------------------------------- * Global server state *----------------------------------------------------------------------------*/ @@ -1427,11 +1442,8 @@ struct redisServer { /* TLS Configuration */ int tls_cluster; int tls_replication; - char *tls_cert_file; - char *tls_key_file; - char *tls_dh_params_file; - char *tls_ca_cert_file; int tls_auth_clients; + redisTLSContextConfig tls_ctx_config; }; typedef struct pubsubPattern { @@ -2371,9 +2383,7 @@ int populateCommandTableParseFlags(struct redisCommand *c, char *strflags); /* TLS stuff */ void tlsInit(void); -int tlsConfigureServer(void); -int tlsConfigure(const char *cert_file, const char *key_file, - const char *dh_params_file, const char *ca_cert_file); +int tlsConfigure(redisTLSContextConfig *ctx_config); #define redisDebug(fmt, ...) \ printf("DEBUG %s:%d > " fmt "\n", __FILE__, __LINE__, __VA_ARGS__) diff --git a/src/tls.c b/src/tls.c index 68d15ef9e..a77fbfe3d 100644 --- a/src/tls.c +++ b/src/tls.c @@ -38,10 +38,57 @@ #include #include +#define REDIS_TLS_PROTO_TLSv1 (1<<0) +#define REDIS_TLS_PROTO_TLSv1_1 (1<<1) +#define REDIS_TLS_PROTO_TLSv1_2 (1<<2) +#define REDIS_TLS_PROTO_TLSv1_3 (1<<3) + +/* Use safe defaults */ +#ifdef TLS1_3_VERSION +#define REDIS_TLS_PROTO_DEFAULT (REDIS_TLS_PROTO_TLSv1_2|REDIS_TLS_PROTO_TLSv1_3) +#else +#define REDIS_TLS_PROTO_DEFAULT (REDIS_TLS_PROTO_TLSv1_2) +#endif + extern ConnectionType CT_Socket; SSL_CTX *redis_tls_ctx; +static int parseProtocolsConfig(const char *str) { + int i, count = 0; + int protocols = 0; + + if (!str) return REDIS_TLS_PROTO_DEFAULT; + sds *tokens = sdssplitlen(str, strlen(str), " ", 1, &count); + + if (!tokens) { + serverLog(LL_WARNING, "Invalid tls-protocols configuration string"); + return -1; + } + for (i = 0; i < count; i++) { + if (!strcasecmp(tokens[i], "tlsv1")) protocols |= REDIS_TLS_PROTO_TLSv1; + else if (!strcasecmp(tokens[i], "tlsv1.1")) protocols |= REDIS_TLS_PROTO_TLSv1_1; + else if (!strcasecmp(tokens[i], "tlsv1.2")) protocols |= REDIS_TLS_PROTO_TLSv1_2; + else if (!strcasecmp(tokens[i], "tlsv1.3")) { +#ifdef TLS1_3_VERSION + protocols |= REDIS_TLS_PROTO_TLSv1_3; +#else + serverLog(LL_WARNING, "TLSv1.3 is specified in tls-protocols but not supported by OpenSSL."); + protocols = -1; + break; +#endif + } else { + serverLog(LL_WARNING, "Invalid tls-protocols specified. " + "Use a combination of 'TLSv1', 'TLSv1.1', 'TLSv1.2' and 'TLSv1.3'."); + protocols = -1; + break; + } + } + sdsfreesplitres(tokens, count); + + return protocols; +} + /* list of connections with pending data already read from the socket, but not * served to the reader yet. */ static list *pending_list = NULL; @@ -56,78 +103,109 @@ void tlsInit(void) { } pending_list = listCreate(); -} -int tlsConfigureServer(void) { - return tlsConfigure(server.tls_cert_file, server.tls_key_file, - server.tls_dh_params_file, server.tls_ca_cert_file); + /* Server configuration */ + server.tls_auth_clients = 1; /* Secure by default */ } /* Attempt to configure/reconfigure TLS. This operation is atomic and will * leave the SSL_CTX unchanged if fails. */ -int tlsConfigure(const char *cert_file, const char *key_file, - const char *dh_params_file, const char *ca_cert_file) { - +int tlsConfigure(redisTLSContextConfig *ctx_config) { char errbuf[256]; SSL_CTX *ctx = NULL; - if (!cert_file) { + if (!ctx_config->cert_file) { serverLog(LL_WARNING, "No tls-cert-file configured!"); goto error; } - if (!key_file) { + if (!ctx_config->key_file) { serverLog(LL_WARNING, "No tls-key-file configured!"); goto error; } - if (!ca_cert_file) { + if (!ctx_config->ca_cert_file) { serverLog(LL_WARNING, "No tls-ca-cert-file configured!"); goto error; } - ctx = SSL_CTX_new(TLS_method()); + ctx = SSL_CTX_new(SSLv23_method()); + + SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3); + SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE); + +#ifdef SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS + SSL_CTX_set_options(ctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS); +#endif + + int protocols = parseProtocolsConfig(ctx_config->protocols); + if (protocols == -1) goto error; + + if (!(protocols & REDIS_TLS_PROTO_TLSv1)) + SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1); + if (!(protocols & REDIS_TLS_PROTO_TLSv1_1)) + SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_1); +#ifdef SSL_OP_NO_TLSv1_2 + if (!(protocols & REDIS_TLS_PROTO_TLSv1_2)) + SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_2); +#endif +#ifdef SSL_OP_NO_TLSv1_3 + if (!(protocols & REDIS_TLS_PROTO_TLSv1_3)) + SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1_3); +#endif + +#ifdef SSL_OP_NO_COMPRESSION + SSL_CTX_set_options(ctx, SSL_OP_NO_COMPRESSION); +#endif + +#ifdef SSL_OP_NO_CLIENT_RENEGOTIATION + SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_CLIENT_RENEGOTIATION); +#endif + + if (ctx_config->prefer_server_ciphers) + SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); + SSL_CTX_set_mode(ctx, SSL_MODE_ENABLE_PARTIAL_WRITE|SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER); SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); SSL_CTX_set_ecdh_auto(ctx, 1); - if (SSL_CTX_use_certificate_file(ctx, cert_file, SSL_FILETYPE_PEM) <= 0) { + if (SSL_CTX_use_certificate_file(ctx, ctx_config->cert_file, SSL_FILETYPE_PEM) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - serverLog(LL_WARNING, "Failed to load certificate: %s: %s", cert_file, errbuf); + serverLog(LL_WARNING, "Failed to load certificate: %s: %s", ctx_config->cert_file, errbuf); goto error; } - if (SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM) <= 0) { + if (SSL_CTX_use_PrivateKey_file(ctx, ctx_config->key_file, SSL_FILETYPE_PEM) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - serverLog(LL_WARNING, "Failed to load private key: %s: %s", key_file, errbuf); + serverLog(LL_WARNING, "Failed to load private key: %s: %s", ctx_config->key_file, errbuf); goto error; } - if (SSL_CTX_load_verify_locations(ctx, ca_cert_file, NULL) <= 0) { + if (SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, NULL) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - serverLog(LL_WARNING, "Failed to load CA certificate(s) file: %s: %s", ca_cert_file, errbuf); + serverLog(LL_WARNING, "Failed to load CA certificate(s) file: %s: %s", ctx_config->ca_cert_file, errbuf); goto error; } - if (dh_params_file) { - FILE *dhfile = fopen(dh_params_file, "r"); + if (ctx_config->dh_params_file) { + FILE *dhfile = fopen(ctx_config->dh_params_file, "r"); DH *dh = NULL; if (!dhfile) { - serverLog(LL_WARNING, "Failed to load %s: %s", dh_params_file, strerror(errno)); + serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno)); goto error; } dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); fclose(dhfile); if (!dh) { - serverLog(LL_WARNING, "%s: failed to read DH params.", dh_params_file); + serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); goto error; } if (SSL_CTX_set_tmp_dh(ctx, dh) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", dh_params_file, errbuf); + serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf); DH_free(dh); goto error; } @@ -402,7 +480,7 @@ static void tlsHandleEvent(tls_connection *conn, int mask) { * risk of not calling the read handler again, make sure to add it * to a list of pending connection that should be handled anyway. */ if ((mask & AE_READABLE)) { - if (SSL_has_pending(conn->ssl)) { + if (SSL_pending(conn->ssl) > 0) { if (!conn->pending_list_node) { listAddNodeTail(pending_list, conn); conn->pending_list_node = listLast(pending_list); @@ -704,16 +782,8 @@ void tlsProcessPendingData() { void tlsInit(void) { } -int tlsConfigure(const char *cert_file, const char *key_file, - const char *dh_params_file, const char *ca_cert_file) { - UNUSED(cert_file); - UNUSED(key_file); - UNUSED(dh_params_file); - UNUSED(ca_cert_file); - return C_OK; -} - -int tlsConfigureServer(void) { +int tlsConfigure(redisTLSContextConfig *ctx_config) { + UNUSED(ctx_config); return C_OK; } @@ -723,6 +793,7 @@ connection *connCreateTLS(void) { connection *connCreateAcceptedTLS(int fd, int require_auth) { UNUSED(fd); + UNUSED(require_auth); return NULL; } diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 192137e87..626a3dbb9 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -504,11 +504,15 @@ start_server {tags {"repl"}} { $master config set repl-diskless-sync-delay 1 set master_host [srv 0 host] set master_port [srv 0 port] + set master_pid [srv 0 pid] # put enough data in the db that the rdb file will be bigger than the socket buffers # and since we'll have key-load-delay of 100, 10000 keys will take at least 1 second # we also need the replica to process requests during transfer (which it does only once in 2mb) $master debug populate 10000 test 10000 $master config set rdbcompression no + # If running on Linux, we also measure utime/stime to detect possible I/O handling issues + set os [catch {exec unamee}] + set measure_time [expr {$os == "Linux"} ? 1 : 0] foreach all_drop {no slow fast all} { test "diskless $all_drop replicas drop during rdb pipe" { set replicas {} @@ -533,9 +537,11 @@ start_server {tags {"repl"}} { # using the log file since the replica only responds to INFO once in 2mb wait_for_log_message -1 "*Loading DB in memory*" 8 800 10 - set master_statfile [format "/proc/%s/stat" [srv -2 pid]] - set master_start_metrics [get_cpu_metrics $master_statfile] - set start_time [clock seconds] + if {$measure_time} { + set master_statfile "/proc/$master_pid/stat" + set master_start_metrics [get_cpu_metrics $master_statfile] + set start_time [clock seconds] + } # wait a while so that the pipe socket writer will be # blocked on write (since replica 0 is slow to read from the socket) @@ -573,23 +579,25 @@ start_server {tags {"repl"}} { } # make sure we don't have a busy loop going thought epoll_wait - set master_end_metrics [get_cpu_metrics $master_statfile] - set time_elapsed [expr {[clock seconds]-$start_time}] - set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] - set master_utime [lindex $master_cpu 0] - set master_stime [lindex $master_cpu 1] - if {$::verbose} { - puts "elapsed: $time_elapsed" - puts "master utime: $master_utime" - puts "master stime: $master_stime" - } - if {$all_drop == "all" || $all_drop == "slow"} { - assert {$master_utime < 30} - assert {$master_stime < 30} - } - if {$all_drop == "none" || $all_drop == "fast"} { - assert {$master_utime < 15} - assert {$master_stime < 15} + if {$measure_time} { + set master_end_metrics [get_cpu_metrics $master_statfile] + set time_elapsed [expr {[clock seconds]-$start_time}] + set master_cpu [compute_cpu_usage $master_start_metrics $master_end_metrics] + set master_utime [lindex $master_cpu 0] + set master_stime [lindex $master_cpu 1] + if {$::verbose} { + puts "elapsed: $time_elapsed" + puts "master utime: $master_utime" + puts "master stime: $master_stime" + } + if {$all_drop == "all" || $all_drop == "slow"} { + assert {$master_utime < 70} + assert {$master_stime < 70} + } + if {$all_drop == "none" || $all_drop == "fast"} { + assert {$master_utime < 15} + assert {$master_stime < 15} + } } # verify the data integrity diff --git a/tests/sentinel/run.tcl b/tests/sentinel/run.tcl index 9a2fcfb49..996af906a 100644 --- a/tests/sentinel/run.tcl +++ b/tests/sentinel/run.tcl @@ -6,6 +6,7 @@ cd tests/sentinel source ../instances.tcl set ::instances_count 5 ; # How many instances we use at max. +set ::tlsdir "../../tls" proc main {} { parse_options diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index be6a11c7a..a90ac7f29 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -39,13 +39,14 @@ array set ::redis::callback {} array set ::redis::state {} ;# State in non-blocking reply reading array set ::redis::statestack {} ;# Stack of states, for nested mbulks -proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0}} { +proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}}} { if {$tls} { package require tls ::tls::init \ -cafile "$::tlsdir/ca.crt" \ -certfile "$::tlsdir/redis.crt" \ - -keyfile "$::tlsdir/redis.key" + -keyfile "$::tlsdir/redis.key" \ + {*}$tlsoptions set fd [::tls::socket $server $port] } else { set fd [socket $server $port] diff --git a/tests/unit/tls.tcl b/tests/unit/tls.tcl index 58acdb6a9..950f65557 100644 --- a/tests/unit/tls.tcl +++ b/tests/unit/tls.tcl @@ -14,12 +14,92 @@ start_server {tags {"tls"}} { catch {$s PING} e assert_match {*error*} $e - set resp [r CONFIG SET tls-auth-clients no] + r CONFIG SET tls-auth-clients no set s [redis [srv 0 host] [srv 0 port]] ::tls::import [$s channel] catch {$s PING} e assert_match {PONG} $e - } {} + + r CONFIG SET tls-auth-clients yes + } + + test {TLS: Verify tls-protocols behaves as expected} { + r CONFIG SET tls-protocols TLSv1 + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 0}] + catch {$s PING} e + assert_match {*I/O error*} $e + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1 1}] + catch {$s PING} e + assert_match {PONG} $e + + r CONFIG SET tls-protocols TLSv1.1 + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 0}] + catch {$s PING} e + assert_match {*I/O error*} $e + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.1 1}] + catch {$s PING} e + assert_match {PONG} $e + + r CONFIG SET tls-protocols TLSv1.2 + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.2 0}] + catch {$s PING} e + assert_match {*I/O error*} $e + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-tls1.2 1}] + catch {$s PING} e + assert_match {PONG} $e + + r CONFIG SET tls-protocols "" + } + + test {TLS: Verify tls-ciphers behaves as expected} { + r CONFIG SET tls-protocols TLSv1.2 + r CONFIG SET tls-ciphers "DEFAULT:-AES128-SHA256" + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-cipher "-ALL:AES128-SHA256"}] + catch {$s PING} e + assert_match {*I/O error*} $e + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-cipher "-ALL:AES256-SHA256"}] + catch {$s PING} e + assert_match {PONG} $e + + r CONFIG SET tls-ciphers "DEFAULT" + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-cipher "-ALL:AES128-SHA256"}] + catch {$s PING} e + assert_match {PONG} $e + + r CONFIG SET tls-protocols "" + r CONFIG SET tls-ciphers "DEFAULT" + } + + test {TLS: Verify tls-prefer-server-ciphers behaves as expected} { + r CONFIG SET tls-protocols TLSv1.2 + r CONFIG SET tls-ciphers "AES128-SHA256:AES256-SHA256" + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-cipher "AES256-SHA256:AES128-SHA256"}] + catch {$s PING} e + assert_match {PONG} $e + + assert_equal "AES256-SHA256" [dict get [::tls::status [$s channel]] cipher] + + r CONFIG SET tls-prefer-server-ciphers yes + + set s [redis [srv 0 host] [srv 0 port] 0 1 {-cipher "AES256-SHA256:AES128-SHA256"}] + catch {$s PING} e + assert_match {PONG} $e + + assert_equal "AES128-SHA256" [dict get [::tls::status [$s channel]] cipher] + + r CONFIG SET tls-protocols "" + r CONFIG SET tls-ciphers "DEFAULT" + } } } From 514e5aa8efd998cfebbdea373f88ec7702be98cb Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Tue, 8 Oct 2019 17:57:05 +0300 Subject: [PATCH 171/320] TLS: Improve CA certifiate configuration options. This adds support for explicit configuration of a CA certs directory (in addition to the previously supported bundle file). For redis-cli, if no explicit CA configuration is supplied the system-wide default configuration will be adopted. --- redis.conf | 6 ++-- src/config.c | 15 +++++++++ src/redis-cli.c | 83 ++++++++++++++++++++++++++++++++++++++++++++----- src/server.h | 1 + src/tls.c | 8 ++--- 5 files changed, 100 insertions(+), 13 deletions(-) diff --git a/redis.conf b/redis.conf index 1f966badb..408426f15 100644 --- a/redis.conf +++ b/redis.conf @@ -148,10 +148,12 @@ tcp-keepalive 300 # # tls-dh-params-file redis.dh -# Configure a CA certificate(s) bundle to authenticate TLS/SSL clients and -# peers. +# Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL +# clients and peers. Redis requires an explicit configuration of at least one +# of these, and will not implicitly use the system wide configuration. # # tls-ca-cert-file ca.crt +# tls-ca-cert-dir /etc/ssl/certs # If TLS/SSL clients are required to authenticate using a client side # certificate, use this directive. diff --git a/src/config.c b/src/config.c index bb04cb45b..fff522815 100644 --- a/src/config.c +++ b/src/config.c @@ -821,6 +821,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"tls-ca-cert-file") && argc == 2) { zfree(server.tls_ctx_config.ca_cert_file); server.tls_ctx_config.ca_cert_file = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"tls-ca-cert-dir") && argc == 2) { + zfree(server.tls_ctx_config.ca_cert_dir); + server.tls_ctx_config.ca_cert_dir = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"tls-protocols") && argc >= 2) { zfree(server.tls_ctx_config.protocols); server.tls_ctx_config.protocols = zstrdup(argv[1]); @@ -1319,6 +1322,16 @@ void configSetCommand(client *c) { } zfree(server.tls_ctx_config.ca_cert_file); server.tls_ctx_config.ca_cert_file = zstrdup(o->ptr); + } config_set_special_field("tls-ca-cert-dir") { + redisTLSContextConfig tmpctx = server.tls_ctx_config; + tmpctx.ca_cert_dir = (char *) o->ptr; + if (tlsConfigure(&tmpctx) == C_ERR) { + addReplyError(c, + "Unable to configure tls-ca-cert-dir. Check server logs."); + return; + } + zfree(server.tls_ctx_config.ca_cert_dir); + server.tls_ctx_config.ca_cert_dir = zstrdup(o->ptr); } config_set_bool_field("tls-auth-clients", server.tls_auth_clients) { } config_set_bool_field("tls-replication", server.tls_replication) { } config_set_bool_field("tls-cluster", server.tls_cluster) { @@ -1439,6 +1452,7 @@ void configGetCommand(client *c) { config_get_string_field("tls-key-file",server.tls_ctx_config.key_file); config_get_string_field("tls-dh-params-file",server.tls_ctx_config.dh_params_file); config_get_string_field("tls-ca-cert-file",server.tls_ctx_config.ca_cert_file); + config_get_string_field("tls-ca-cert-dir",server.tls_ctx_config.ca_cert_dir); config_get_string_field("tls-protocols",server.tls_ctx_config.protocols); config_get_string_field("tls-ciphers",server.tls_ctx_config.ciphers); config_get_string_field("tls-ciphersuites",server.tls_ctx_config.ciphersuites); @@ -2347,6 +2361,7 @@ int rewriteConfig(char *path) { rewriteConfigStringOption(state,"tls-key-file",server.tls_ctx_config.key_file,NULL); rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_ctx_config.dh_params_file,NULL); rewriteConfigStringOption(state,"tls-ca-cert-file",server.tls_ctx_config.ca_cert_file,NULL); + rewriteConfigStringOption(state,"tls-ca-cert-dir",server.tls_ctx_config.ca_cert_dir,NULL); rewriteConfigStringOption(state,"tls-protocols",server.tls_ctx_config.protocols,NULL); rewriteConfigStringOption(state,"tls-ciphers",server.tls_ctx_config.ciphers,NULL); rewriteConfigStringOption(state,"tls-ciphersuites",server.tls_ctx_config.ciphersuites,NULL); diff --git a/src/redis-cli.c b/src/redis-cli.c index c7145927a..01211ee24 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -48,6 +48,7 @@ #include #ifdef USE_OPENSSL +#include #include #endif #include /* use sds.h from hiredis, so that only one set of sds functions will be present in the binary */ @@ -194,6 +195,7 @@ static struct config { int tls; char *sni; char *cacert; + char *cacertdir; char *cert; char *key; long repeat; @@ -762,9 +764,61 @@ static int cliSelect(void) { /* Wrapper around redisSecureConnection to avoid hiredis_ssl dependencies if * not building with TLS support. */ -static int cliSecureConnection(redisContext *c) { +static int cliSecureConnection(redisContext *c, const char **err) { #ifdef USE_OPENSSL - return redisSecureConnection(c, config.cacert, config.cert, config.key, config.sni); + static SSL_CTX *ssl_ctx = NULL; + + if (!ssl_ctx) { + ssl_ctx = SSL_CTX_new(SSLv23_client_method()); + if (!ssl_ctx) { + *err = "Failed to create SSL_CTX"; + goto error; + } + + SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + + if (config.cacert || config.cacertdir) { + if (!SSL_CTX_load_verify_locations(ssl_ctx, config.cacert, config.cacertdir)) { + *err = "Invalid CA Certificate File/Directory"; + goto error; + } + } else { + if (!SSL_CTX_set_default_verify_paths(ssl_ctx)) { + *err = "Failed to use default CA paths"; + goto error; + } + } + + if (config.cert && !SSL_CTX_use_certificate_chain_file(ssl_ctx, config.cert)) { + *err = "Invalid client certificate"; + goto error; + } + + if (config.key && !SSL_CTX_use_PrivateKey_file(ssl_ctx, config.key, SSL_FILETYPE_PEM)) { + *err = "Invalid private key"; + goto error; + } + } + + SSL *ssl = SSL_new(ssl_ctx); + if (!ssl) { + *err = "Failed to create SSL object"; + return REDIS_ERR; + } + + if (config.sni && !SSL_set_tlsext_host_name(ssl, config.sni)) { + *err = "Failed to configure SNI"; + SSL_free(ssl); + return REDIS_ERR; + } + + return redisInitiateSSL(c, ssl); + +error: + SSL_CTX_free(ssl_ctx); + ssl_ctx = NULL; + return REDIS_ERR; #else (void) c; return REDIS_OK; @@ -788,9 +842,9 @@ static int cliConnect(int flags) { } if (!context->err && config.tls) { - if (cliSecureConnection(context) == REDIS_ERR && !context->err) { - /* TODO: this check should be redundant, redis-cli should set err=1 */ - fprintf(stderr, "Could not negotiate a TLS connection.\n"); + const char *err = NULL; + if (cliSecureConnection(context, &err) == REDIS_ERR && err) { + fprintf(stderr, "Could not negotiate a TLS connection: %s\n", err); context = NULL; redisFree(context); return REDIS_ERR; @@ -1277,7 +1331,11 @@ static redisReply *reconnectingRedisCommand(redisContext *c, const char *fmt, .. redisFree(c); c = redisConnect(config.hostip,config.hostport); if (!c->err && config.tls) { - cliSecureConnection(c); + const char *err = NULL; + if (cliSecureConnection(c, &err) == REDIS_ERR && err) { + fprintf(stderr, "TLS Error: %s\n", err); + exit(1); + } } usleep(1000000); } @@ -1473,6 +1531,8 @@ static int parseOptions(int argc, char **argv) { config.tls = 1; } else if (!strcmp(argv[i],"--sni")) { config.sni = argv[++i]; + } else if (!strcmp(argv[i],"--cacertdir")) { + config.cacertdir = argv[++i]; } else if (!strcmp(argv[i],"--cacert")) { config.cacert = argv[++i]; } else if (!strcmp(argv[i],"--cert")) { @@ -1571,6 +1631,9 @@ static void usage(void) { #ifdef USE_OPENSSL " --tls Establish a secure TLS connection.\n" " --cacert CA Certificate file to verify with.\n" +" --cacertdir Directory where trusted CA certificates are stored.\n" +" If neither cacert nor cacertdir are specified, the default\n" +" system-wide trusted root certs configuration will apply.\n" " --cert Client certificate to authenticate with.\n" " --key Private key file to authenticate with.\n" #endif @@ -2390,7 +2453,13 @@ static int clusterManagerNodeConnect(clusterManagerNode *node) { if (node->context) redisFree(node->context); node->context = redisConnect(node->ip, node->port); if (!node->context->err && config.tls) { - cliSecureConnection(node->context); + const char *err = NULL; + if (cliSecureConnection(node->context, &err) == REDIS_ERR && err) { + fprintf(stderr,"TLS Error: %s\n", err); + redisFree(node->context); + node->context = NULL; + return 0; + } } if (node->context->err) { fprintf(stderr,"Could not connect to Redis at "); diff --git a/src/server.h b/src/server.h index 5c7849732..85e5de97a 100644 --- a/src/server.h +++ b/src/server.h @@ -1038,6 +1038,7 @@ typedef struct redisTLSContextConfig { char *key_file; char *dh_params_file; char *ca_cert_file; + char *ca_cert_dir; char *protocols; char *ciphers; char *ciphersuites; diff --git a/src/tls.c b/src/tls.c index a77fbfe3d..5fac6902b 100644 --- a/src/tls.c +++ b/src/tls.c @@ -125,8 +125,8 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { goto error; } - if (!ctx_config->ca_cert_file) { - serverLog(LL_WARNING, "No tls-ca-cert-file configured!"); + if (!ctx_config->ca_cert_file && !ctx_config->ca_cert_dir) { + serverLog(LL_WARNING, "Either tls-ca-cert-file or tls-ca-cert-dir must be configured!"); goto error; } @@ -182,9 +182,9 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { goto error; } - if (SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, NULL) <= 0) { + if (SSL_CTX_load_verify_locations(ctx, ctx_config->ca_cert_file, ctx_config->ca_cert_dir) <= 0) { ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); - serverLog(LL_WARNING, "Failed to load CA certificate(s) file: %s: %s", ctx_config->ca_cert_file, errbuf); + serverLog(LL_WARNING, "Failed to configure CA certificate(s) file/directory: %s", errbuf); goto error; } From bdca9e882d1619bde19cbad78549f3167ddc3d41 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 8 Oct 2019 17:02:52 +0200 Subject: [PATCH 172/320] Geo: output 10 chars of geohash, not 11. This does not limit the actual precision, because the last digit bits were garbage, and the shift value became even negative in the last iteration. --- src/geo.c | 8 ++++---- tests/unit/geo.tcl | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/geo.c b/src/geo.c index f47f4ee22..049335a4f 100644 --- a/src/geo.c +++ b/src/geo.c @@ -734,14 +734,14 @@ void geohashCommand(client *c) { r[1].max = 90; geohashEncode(&r[0],&r[1],xy[0],xy[1],26,&hash); - char buf[12]; + char buf[11]; int i; - for (i = 0; i < 11; i++) { + for (i = 0; i < 10; i++) { int idx = (hash.bits >> (52-((i+1)*5))) & 0x1f; buf[i] = geoalphabet[idx]; } - buf[11] = '\0'; - addReplyBulkCBuffer(c,buf,11); + buf[10] = '\0'; + addReplyBulkCBuffer(c,buf,10); } } } diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl index 49e421ee9..76b9bda38 100644 --- a/tests/unit/geo.tcl +++ b/tests/unit/geo.tcl @@ -129,7 +129,7 @@ start_server {tags {"geo"}} { r del points r geoadd points -5.6 42.6 test lindex [r geohash points test] 0 - } {ezs42e44yx0} + } {ezs42e44yx} test {GEOPOS simple} { r del points From c3562cf13493f0728354d8814fff9cf19912f709 Mon Sep 17 00:00:00 2001 From: omg-by <504094596@qq.com> Date: Thu, 10 Oct 2019 00:43:28 +0800 Subject: [PATCH 173/320] there should is AUTH && HELLO non authenticated state. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index e089865f8..a9b6b650e 100644 --- a/src/server.c +++ b/src/server.c @@ -3366,7 +3366,7 @@ int processCommand(client *c) { !c->authenticated; if (auth_required) { /* AUTH and HELLO are valid even in non authenticated state. */ - if (c->cmd->proc != authCommand || c->cmd->proc == helloCommand) { + if (c->cmd->proc != authCommand && c->cmd->proc != helloCommand) { flagTransaction(c); addReply(c,shared.noautherr); return C_OK; From 46c1377c83cfc4549ab10a5a8c1e9ec66af2fc30 Mon Sep 17 00:00:00 2001 From: Daniel Dai <764122422@qq.com> Date: Wed, 9 Oct 2019 14:15:31 -0400 Subject: [PATCH 174/320] update typo --- tests/integration/replication.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 5d32555b0..1c18582c5 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -319,7 +319,7 @@ start_server {tags {"repl"}} { } } -test {slave fails full sync and diskless load swapdb recoveres it} { +test {slave fails full sync and diskless load swapdb recovers it} { start_server {tags {"repl"}} { set slave [srv 0 client] set slave_host [srv 0 host] From 33c424b240f469da5457c6f82a4de1d037f13aca Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 10 Oct 2019 09:47:48 +0200 Subject: [PATCH 175/320] Fix usage of server.stream_node_max_* --- src/t_stream.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 9e7d3d126..ea9a620f1 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -242,17 +242,17 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ * the current node is full. */ if (lp != NULL) { if (server.stream_node_max_bytes && - lp_bytes > server.stream_node_max_bytes) + lp_bytes >= server.stream_node_max_bytes) { lp = NULL; } else if (server.stream_node_max_entries) { int64_t count = lpGetInteger(lpFirst(lp)); - if (count > server.stream_node_max_entries) lp = NULL; + if (count >= server.stream_node_max_entries) lp = NULL; } } int flags = STREAM_ITEM_FLAG_NONE; - if (lp == NULL || lp_bytes > server.stream_node_max_bytes) { + if (lp == NULL || lp_bytes >= server.stream_node_max_bytes) { master_id = id; streamEncodeID(rax_key,&id); /* Create the listpack having the master entry ID and fields. */ From 9aba7564d6be947e6c48427291ecd64339ff336d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Oct 2019 10:23:34 +0200 Subject: [PATCH 176/320] Cluster: fix memory leak of cached master. This is what happened: 1. Instance starts, is a slave in the cluster configuration, but actually server.masterhost is not set, so technically the instance is acting like a master. 2. loadDataFromDisk() calls replicationCacheMasterUsingMyself() even if the instance is a master, in the case it is logically a slave and the cluster is enabled. So now we have a cached master even if the instance is practically configured as a master (from the POV of server.masterhost value and so forth). 3. clusterCron() sees that the instance requires to replicate from its master, because logically it is a slave, so it calls replicationSetMaster() that will in turn call replicationCacheMasterUsingMyself(): before this commit, this call would overwrite the old cached master, creating a memory leak. --- src/replication.c | 5 ++++- src/server.c | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 76090a9e5..081212761 100644 --- a/src/replication.c +++ b/src/replication.c @@ -2182,7 +2182,10 @@ void replicationSetMaster(char *ip, int port) { cancelReplicationHandshake(); /* Before destroying our master state, create a cached master using * our own parameters, to later PSYNC with the new master. */ - if (was_master) replicationCacheMasterUsingMyself(); + if (was_master) { + replicationDiscardCachedMaster(); + replicationCacheMasterUsingMyself(); + } server.repl_state = REPL_STATE_CONNECT; } diff --git a/src/server.c b/src/server.c index e089865f8..9392ffb8e 100644 --- a/src/server.c +++ b/src/server.c @@ -4719,12 +4719,14 @@ void loadDataFromDisk(void) { (float)(ustime()-start)/1000000); /* Restore the replication ID / offset from the RDB file. */ - if ((server.masterhost || (server.cluster_enabled && nodeIsSlave(server.cluster->myself)))&& + if ((server.masterhost || + (server.cluster_enabled && + nodeIsSlave(server.cluster->myself))) && rsi.repl_id_is_set && rsi.repl_offset != -1 && /* Note that older implementations may save a repl_stream_db - * of -1 inside the RDB file in a wrong way, see more information - * in function rdbPopulateSaveInfo. */ + * of -1 inside the RDB file in a wrong way, see more + * information in function rdbPopulateSaveInfo. */ rsi.repl_stream_db != -1) { memcpy(server.replid,rsi.repl_id,sizeof(server.replid)); From 0ec3cbdb8d24323806452b5c7c1c0a67d0815141 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Oct 2019 15:22:42 +0200 Subject: [PATCH 177/320] Test: fix implementation-dependent test after code change. --- tests/unit/type/stream.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index ae6c2d7b8..a7415ae8d 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -355,12 +355,12 @@ start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries r XADD mystream * xitem v } r XTRIM mystream MAXLEN ~ 85 - assert {[r xlen mystream] == 89} + assert {[r xlen mystream] == 90} r config set stream-node-max-entries 1 r debug loadaof r XADD mystream * xitem v incr j - assert {[r xlen mystream] == 90} + assert {[r xlen mystream] == 91} } } From d2aee5e4fed92b80ba913e2ee8a332df208765a3 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Tue, 15 Oct 2019 15:24:32 +0300 Subject: [PATCH 178/320] Fix compile warnings when BUILD_TLS=no. --- src/redis-cli.c | 1 + src/sentinel.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 01211ee24..417f1b1a9 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -821,6 +821,7 @@ error: return REDIS_ERR; #else (void) c; + (void) err; return REDIS_OK; #endif } diff --git a/src/sentinel.c b/src/sentinel.c index 92869805d..0490db4e9 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2005,7 +2005,7 @@ void sentinelSetClientName(sentinelRedisInstance *ri, redisAsyncContext *c, char static int instanceLinkNegotiateTLS(redisAsyncContext *context) { #ifndef USE_OPENSSL - (void) link; + (void) context; #else if (!redis_tls_ctx) return C_ERR; SSL *ssl = SSL_new(redis_tls_ctx); From 3a990ebcb4f5c9aa3e16d4a5dbf515b72ab9e9af Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Tue, 15 Oct 2019 17:20:58 +0300 Subject: [PATCH 179/320] Conns: write() 0 retval should not trigger error. --- src/connection.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/connection.c b/src/connection.c index 85bf572ad..5a4b48a3e 100644 --- a/src/connection.c +++ b/src/connection.c @@ -162,9 +162,7 @@ static void connSocketClose(connection *conn) { static int connSocketWrite(connection *conn, const void *data, size_t data_len) { int ret = write(conn->fd, data, data_len); - if (!ret) { - conn->state = CONN_STATE_CLOSED; - } else if (ret < 0 && errno != EAGAIN) { + if (ret < 0 && errno != EAGAIN) { conn->last_errno = errno; conn->state = CONN_STATE_ERROR; } From e17ac77a826fb5dee96fb8de4a092eb776bdeea5 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Tue, 15 Oct 2019 17:21:33 +0300 Subject: [PATCH 180/320] Code review minor changes (names, comments). --- src/ae.c | 2 +- src/ae.h | 2 +- src/connection.c | 9 +++++++-- src/connhelpers.h | 25 +++++++++++++++++++++++++ src/rdb.c | 4 ++-- src/server.c | 2 +- 6 files changed, 37 insertions(+), 7 deletions(-) diff --git a/src/ae.c b/src/ae.c index 54c0d994e..2c1dae512 100644 --- a/src/ae.c +++ b/src/ae.c @@ -99,7 +99,7 @@ int aeGetSetSize(aeEventLoop *eventLoop) { } /* Tells the next iteration/s of the event processing to set timeout of 0. */ -void aeDontWait(aeEventLoop *eventLoop, int noWait) { +void aeSetDontWait(aeEventLoop *eventLoop, int noWait) { if (noWait) eventLoop->flags |= AE_DONT_WAIT; else diff --git a/src/ae.h b/src/ae.h index bbb43fe6e..9acd72434 100644 --- a/src/ae.h +++ b/src/ae.h @@ -129,6 +129,6 @@ void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep void aeSetAfterSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *aftersleep); int aeGetSetSize(aeEventLoop *eventLoop); int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); -void aeDontWait(aeEventLoop *eventLoop, int noWait); +void aeSetDontWait(aeEventLoop *eventLoop, int noWait); #endif diff --git a/src/connection.c b/src/connection.c index 5a4b48a3e..58d86c31b 100644 --- a/src/connection.c +++ b/src/connection.c @@ -191,6 +191,11 @@ static int connSocketAccept(connection *conn, ConnectionCallbackFunc accept_hand /* Register a write handler, to be called when the connection is writable. * If NULL, the existing handler is removed. + * + * The barrier flag indicates a write barrier is requested, resulting with + * CONN_FLAG_WRITE_BARRIER set. This will ensure that the write handler is + * always called before and not after the read handler in a single event + * loop. */ static int connSocketSetWriteHandler(connection *conn, ConnectionCallbackFunc func, int barrier) { if (func == conn->write_handler) return C_OK; @@ -250,7 +255,7 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD } /* Normally we execute the readable event first, and the writable - * event laster. This is useful as sometimes we may be able + * event later. This is useful as sometimes we may be able * to serve the reply of a query immediately after processing the * query. * @@ -258,7 +263,7 @@ static void connSocketEventHandler(struct aeEventLoop *el, int fd, void *clientD * asking us to do the reverse: never fire the writable event * after the readable. In such a case, we invert the calls. * This is useful when, for instance, we want to do things - * in the beforeSleep() hook, like fsynching a file to disk, + * in the beforeSleep() hook, like fsync'ing a file to disk, * before replying to a client. */ int invert = conn->flags & CONN_FLAG_WRITE_BARRIER; diff --git a/src/connhelpers.h b/src/connhelpers.h index 2ceccd085..f237c9b1d 100644 --- a/src/connhelpers.h +++ b/src/connhelpers.h @@ -33,10 +33,29 @@ #include "connection.h" +/* These are helper functions that are common to different connection + * implementations (currently sockets in connection.c and TLS in tls.c). + * + * Currently helpers implement the mechanisms for invoking connection + * handlers, tracking in-handler states and dealing with deferred + * destruction (if invoked by a handler). + */ + +/* Called whenever a handler is invoked on a connection and sets the + * CONN_FLAG_IN_HANDLER flag to indicate we're in a handler context. + * + * An attempt to close a connection while CONN_FLAG_IN_HANDLER is + * set will result with deferred close, i.e. setting the CONN_FLAG_CLOSE_SCHEDULED + * instead of destructing it. + */ static inline void enterHandler(connection *conn) { conn->flags |= CONN_FLAG_IN_HANDLER; } +/* Called whenever a handler returns. This unsets the CONN_FLAG_IN_HANDLER + * flag and performs actual close/destruction if a deferred close was + * scheduled by the handler. + */ static inline int exitHandler(connection *conn) { conn->flags &= ~CONN_FLAG_IN_HANDLER; if (conn->flags & CONN_FLAG_CLOSE_SCHEDULED) { @@ -46,6 +65,12 @@ static inline int exitHandler(connection *conn) { return 1; } +/* Helper for connection implementations to call handlers: + * 1. Mark the handler in use. + * 2. Execute the handler (if set). + * 3. Mark the handler as NOT in use and perform deferred close if was + * requested by the handler at any time. + */ static inline int callHandler(connection *conn, ConnectionCallbackFunc handler) { conn->flags |= CONN_FLAG_IN_HANDLER; if (handler) handler(conn); diff --git a/src/rdb.c b/src/rdb.c index 59f22916b..fd279d3b9 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2365,8 +2365,8 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) { if (server.rdb_pipe_conns) return C_ERR; /* Before to fork, create a pipe that is used to transfer the rdb bytes to - * the parant, we can't let it write directly to the sockets, since in case - * of TLS we must let the parent handle a contineous TLS state when the + * the parent, we can't let it write directly to the sockets, since in case + * of TLS we must let the parent handle a continuous TLS state when the * child terminates and parent takes over. */ if (pipe(pipefds) == -1) return C_ERR; server.rdb_pipe_read = pipefds[0]; diff --git a/src/server.c b/src/server.c index 3de1347cb..ef3f7f326 100644 --- a/src/server.c +++ b/src/server.c @@ -2051,7 +2051,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */ tlsProcessPendingData(); /* If tls still has pending unread data don't sleep at all. */ - aeDontWait(server.el, tlsHasPendingData()); + aeSetDontWait(server.el, tlsHasPendingData()); /* Call the Redis Cluster before sleep function. Note that this function * may change the state of Redis Cluster (from ok to fail or vice versa), From 2255860a223e9fd3669ec4481cb0eaa2e5bad3f9 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 16 Oct 2019 17:31:02 +0300 Subject: [PATCH 181/320] Fix Makefile merge issue. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 2b404bc58..9fc230f94 100644 --- a/src/Makefile +++ b/src/Makefile @@ -172,7 +172,7 @@ endif REDIS_SERVER_NAME=redis-server REDIS_SENTINEL_NAME=redis-sentinel -REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o acl.o gopher.o tracking.o connection.o tls.o sha256.o +REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o gopher.o tracking.o connection.o tls.o sha256.o REDIS_CLI_NAME=redis-cli REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o REDIS_BENCHMARK_NAME=redis-benchmark From 34961ddee82497d9ffe5aaf55aada50189b2dbc8 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 17 Oct 2019 17:49:33 +0200 Subject: [PATCH 182/320] Modules: add new flags to context, replica state + more. --- src/module.c | 32 ++++++++++++++++++++++++++++++++ src/redismodule.h | 17 ++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 0d83817fa..fe497f5dc 100644 --- a/src/module.c +++ b/src/module.c @@ -1551,6 +1551,21 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * * * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * reaching the maxmemory level. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to + * connect with the master. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING: Master -> Replica RDB + * transfer is in progress. + * + * * REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE: The replica has an active link + * with its master. This is the + * contrary of STALE state. + * + * * REDISMODULE_CTX_FLAGS_ACTIVE_CHILD: There is currently some background + * process active (RDB, AUX or module). */ int RM_GetContextFlags(RedisModuleCtx *ctx) { @@ -1593,6 +1608,20 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { flags |= REDISMODULE_CTX_FLAGS_SLAVE; if (server.repl_slave_ro) flags |= REDISMODULE_CTX_FLAGS_READONLY; + + /* Replica state flags. */ + if (server.repl_state == REPL_STATE_CONNECT || + server.repl_state == REPL_STATE_CONNECTING) + { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING; + } else if (server.repl_state == REPL_STATE_TRANSFER) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING; + } else if (server.repl_state == REPL_STATE_CONNECTED) { + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE; + } + + if (server.repl_state != REPL_STATE_CONNECTED) + flags |= REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE; } /* OOM flag. */ @@ -1601,6 +1630,9 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) { if (retval == C_ERR) flags |= REDISMODULE_CTX_FLAGS_OOM; if (level > 0.75) flags |= REDISMODULE_CTX_FLAGS_OOM_WARNING; + /* Presence of children processes. */ + if (hasActiveChildProcess()) flags |= REDISMODULE_CTX_FLAGS_ACTIVE_CHILD; + return flags; } diff --git a/src/redismodule.h b/src/redismodule.h index 851bb8cde..a7e69f7b3 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -89,7 +89,22 @@ #define REDISMODULE_CTX_FLAGS_REPLICATED (1<<12) /* Redis is currently loading either from AOF or RDB. */ #define REDISMODULE_CTX_FLAGS_LOADING (1<<13) - +/* The replica has no link with its master, note that + * there is the inverse flag as well: + * + * REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE + * + * The two flags are exclusive, one or the other can be set. */ +#define REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE (1<<14) +/* The replica is trying to connect with the master. + * (REPL_STATE_CONNECT and REPL_STATE_CONNECTING states) */ +#define REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING (1<<15) +/* THe replica is receiving an RDB file from its master. */ +#define REDISMODULE_CTX_FLAGS_REPLICA_IS_TRANSFERRING (1<<16) +/* The replica is online, receiving updates from its master. */ +#define REDISMODULE_CTX_FLAGS_REPLICA_IS_ONLINE (1<<17) +/* There is currently some background process active. */ +#define REDISMODULE_CTX_FLAGS_ACTIVE_CHILD (1<<18) #define REDISMODULE_NOTIFY_GENERIC (1<<2) /* g */ #define REDISMODULE_NOTIFY_STRING (1<<3) /* $ */ From ea85cda320da0f69e2288e3734b677d1b951a9fb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 17 Oct 2019 18:10:50 +0200 Subject: [PATCH 183/320] Modules: allow to check for AOF loading client. --- src/aof.c | 5 +++-- src/module.c | 10 +++++++++- src/redismodule.h | 2 ++ src/server.h | 5 +++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/aof.c b/src/aof.c index 32684eb89..0e3648ff0 100644 --- a/src/aof.c +++ b/src/aof.c @@ -652,10 +652,11 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ -struct client *createFakeClient(void) { +struct client *createAOFClient(void) { struct client *c = zmalloc(sizeof(*c)); selectDb(c,0); + c->id = CLIENT_ID_AOF; /* So modules can identify it's the AOF client. */ c->conn = NULL; c->name = NULL; c->querybuf = sdsempty(); @@ -729,7 +730,7 @@ int loadAppendOnlyFile(char *filename) { * to the same file we're about to read. */ server.aof_state = AOF_OFF; - fakeClient = createFakeClient(); + fakeClient = createAOFClient(); startLoadingFile(fp, filename); /* Check if this AOF file has an RDB preamble. In that case we need to diff --git a/src/module.c b/src/module.c index fe497f5dc..cf6da9aa1 100644 --- a/src/module.c +++ b/src/module.c @@ -1504,7 +1504,15 @@ int RM_ReplicateVerbatim(RedisModuleCtx *ctx) { * are guaranteed to get IDs greater than any past ID previously seen. * * Valid IDs are from 1 to 2^64-1. If 0 is returned it means there is no way - * to fetch the ID in the context the function was currently called. */ + * to fetch the ID in the context the function was currently called. + * + * After obtaining the ID, it is possible to check if the command execution + * is actually happening in the context of AOF loading, using this macro: + * + * if (RedisModule_IsAOFClient(RedisModule_GetClientId(ctx)) { + * // Handle it differently. + * } + */ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { if (ctx->client == NULL) return 0; return ctx->client->id; diff --git a/src/redismodule.h b/src/redismodule.h index a7e69f7b3..19a9cd897 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -404,6 +404,8 @@ int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); #endif +#define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) + /* This is included inline inside each Redis module. */ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) __attribute__((unused)); static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int apiver) { diff --git a/src/server.h b/src/server.h index 00b54bd35..829d8213e 100644 --- a/src/server.h +++ b/src/server.h @@ -827,6 +827,11 @@ typedef struct user { /* With multiplexing we need to take per-client state. * Clients are taken in a linked list. */ + +#define CLIENT_ID_AOF (UINT64_MAX) /* Reserved ID for the AOF client. If you + need more reserved IDs use UINT64_MAX-1, + -2, ... and so forth. */ + typedef struct client { uint64_t id; /* Client incremental unique ID. */ connection *conn; From 7414be69c86f23a65281009a65a4347a76e19f17 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 17 Oct 2019 18:28:34 +0200 Subject: [PATCH 184/320] module: fix propagation API bug. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index cf6da9aa1..ff5eba787 100644 --- a/src/module.c +++ b/src/module.c @@ -561,9 +561,9 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) { propagate(rop->cmd,rop->dbid,rop->argv,rop->argc,target); } redisOpArrayFree(&server.also_propagate); + /* Restore the previous oparray in case of nexted use of the API. */ + server.also_propagate = ctx->saved_oparray; } - /* Restore the previous oparray in case of nexted use of the API. */ - server.also_propagate = ctx->saved_oparray; } /* Free the context after the user function was called. */ From 82e1417eb7c3dd0256d6693a6f29d67fbc7f53a2 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Sun, 20 Oct 2019 10:04:25 +0300 Subject: [PATCH 185/320] add CI action --- .circleci/config.yml | 16 ---------------- .github/pull.yml | 14 -------------- .github/workflows/ci.yml | 28 ++++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 30 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/pull.yml create mode 100644 .github/workflows/ci.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index fe2a6ea07..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/buildpack-deps - steps: - - checkout - - run: - name: Install dependency - command: sudo apt-get install -y tcl - - run: - name: Build - command: make - - run: - name: Test - command: make test diff --git a/.github/pull.yml b/.github/pull.yml deleted file mode 100644 index 6ccf19ca9..000000000 --- a/.github/pull.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: "1" -rules: - - base: unstable - upstream: antirez:unstable - mergeMethod: merge - - base: 5.0 - upstream: antirez:5.0 - mergeMethod: merge - - base: 4.0 - upstream: antirez:4.0 - mergeMethod: merge - - base: 3.2 - upstream: antirez:3.2 - mergeMethod: merge diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..847abcf02 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: CI + +on: [push, pull_request] + +jobs: + build-ubuntu: + strategy: + matrix: + platform: [ubuntu-latest, ubuntu-16.04] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v1 + - name: make + run: make + - name: test + run: | + sudo apt-get install tcl8.5 + make test + + build-macos-latest: + strategy: + matrix: + platform: [macos-latest, macOS-10.14] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v1 + - name: make + run: make From 58a0ef302014a36f853aaf16347929ef755061c9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 18:39:14 +0200 Subject: [PATCH 186/320] Modules: fix moduleCreateArgvFromUserFormat() casting bug. In 32 bit systems casting to "long" will cut the result to 32 bit. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index ff5eba787..1c09fb7f0 100644 --- a/src/module.c +++ b/src/module.c @@ -2853,7 +2853,7 @@ robj **moduleCreateArgvFromUserFormat(const char *cmdname, const char *fmt, int size_t len = va_arg(ap,size_t); argv[argc++] = createStringObject(buf,len); } else if (*p == 'l') { - long ll = va_arg(ap,long long); + long long ll = va_arg(ap,long long); argv[argc++] = createObject(OBJ_STRING,sdsfromlonglong(ll)); } else if (*p == 'v') { /* A vector of strings */ From 7333e5c5a8fe76289511acf6073d27cb7e3bf0ad Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 18 Oct 2019 18:01:39 +0200 Subject: [PATCH 187/320] Modules hooks: initial design of data structures. --- src/module.c | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 1c09fb7f0..554a949b9 100644 --- a/src/module.c +++ b/src/module.c @@ -65,7 +65,7 @@ struct RedisModule { list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ int options; /* Module options and capabilities. */ - RedisModuleInfoFunc info_cb; /* callback for module to add INFO fields. */ + RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -323,6 +323,43 @@ static struct RedisModuleForkInfo { #define REDISMODULE_ARGV_NO_AOF (1<<1) #define REDISMODULE_ARGV_NO_REPLICAS (1<<2) +/* Server events hooks data structures and defines: this modules API + * allow modules to subscribe to certain events in Redis, such as + * the start and end of an RDB or AOF save, the change of role in replication, + * and similar other events. */ + +#define REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED 0 +#define REDISMODULE_EVENT_ID_RDB_SAVE_START 1 +#define REDISMODULE_EVENT_ID_RDB_SAVE_END 2 +#define REDISMODULE_EVENT_ID_AOF_REWRITE_START 3 +#define REDISMODULE_EVENT_ID_AOF_REWRITE_END 4 +#define REDISMODULE_EVENT_ID_FLUSHDB_START 5 +#define REDISMODULE_EVENT_ID_FLUSHDB_END 6 +#define REDISMODULE_EVENT_ID_LOADING_START 7 +#define REDISMODULE_EVENT_ID_LOADING_END 8 +#define REDISMODULE_EVENT_ID_CLIENT_CONNNECTED 9 +#define REDISMODULE_EVENT_ID_CLIENT_DISCONNECTED 10 +#define REDISMODULE_EVENT_ID_SERVER_SHUTDOWN 11 +#define REDISMODULE_EVENT_ID_REPLICA_ONLINE 12 +#define REDISMODULE_EVENT_ID_REPLICA_OFFLINE 13 +#define REDISMODULE_EVENT_ID_MASTER_LINK_UP 14 +#define REDISMODULE_EVENT_ID_MASTER_LINK_DOWN 15 + +typedef struct RedisModuleEvent { + uint64_t id; /* REDISMODULE_EVENT_ID_... defines. */ + uint64_t dataver; /* Version of the structure we pass as 'data'. */ +} RedisModuleEvent; + +typedef int (*RedisModuleEventCallback)(RedisModuleEvent eid, void *data); + +typedef struct RedisModuleEventListener { + RedisModule *module; + RedisModuleEvent event; + RedisModuleEventCallback callback; +} RedisModuleEventListener; + +list *RedisModule_EventListeners; /* Global list of all the active events. */ + /* -------------------------------------------------------------------------- * Prototypes * -------------------------------------------------------------------------- */ @@ -5614,6 +5651,9 @@ void moduleInitModulesSystem(void) { /* Create the timers radix tree. */ Timers = raxNew(); + /* Setup the event listeners data structures. */ + RedisModule_EventListeners = listCreate(); + /* Our thread-safe contexts GIL must start with already locked: * it is just unlocked when it's safe. */ pthread_mutex_lock(&moduleGIL); From 1f1d25b5a9cc574a555d7c6afbc668d084369965 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Oct 2019 17:16:34 +0200 Subject: [PATCH 188/320] Modules hooks: a first version of events and some API. --- src/module.c | 58 ++++++++++++++++++++++++++++++++++----- src/redismodule.h | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/src/module.c b/src/module.c index 554a949b9..e3267cdbc 100644 --- a/src/module.c +++ b/src/module.c @@ -345,13 +345,6 @@ static struct RedisModuleForkInfo { #define REDISMODULE_EVENT_ID_MASTER_LINK_UP 14 #define REDISMODULE_EVENT_ID_MASTER_LINK_DOWN 15 -typedef struct RedisModuleEvent { - uint64_t id; /* REDISMODULE_EVENT_ID_... defines. */ - uint64_t dataver; /* Version of the structure we pass as 'data'. */ -} RedisModuleEvent; - -typedef int (*RedisModuleEventCallback)(RedisModuleEvent eid, void *data); - typedef struct RedisModuleEventListener { RedisModule *module; RedisModuleEvent event; @@ -5587,6 +5580,57 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { moduleForkInfo.done_handler_user_data = NULL; } +/* -------------------------------------------------------------------------- + * Server hooks implementation + * -------------------------------------------------------------------------- */ + +/* Register to be notified, via a callback, when the specified server event + * happens. The callback is called with the event as argument, and an additional + * argument which is a void pointer and should be cased to a specific type + * that is event-specific (but many events will just use NULL since they do not + * have additional information to pass to the callback). + * + * If the callback is NULL and there was a previous subscription, the module + * will be unsubscribed. If there was a previous subscription and the callback + * is not null, the old callback will be replaced with the new one. + * + * The function returns REDISMODULE_OK if the module was successfully subscrived + * for the specified event. If the API is called from a wrong context then + * REDISMODULE_ERR is returned. */ +int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) { + RedisModuleEventListener *el; + + /* Protect in case of calls from contexts without a module reference. */ + if (ctx->module == NULL) return REDISMODULE_ERR; + + /* Search an event matching this module and event ID. */ + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + el = ln->value; + if (el->module == ctx->module && el->event.id == event.id) + break; /* Matching event found. */ + } + + /* Modify or remove the event listener if we already had one. */ + if (ln) { + if (callback == NULL) + listDelNode(RedisModule_EventListeners,ln); + else + el->callback = callback; /* Update the callback with the new one. */ + return REDISMODULE_OK; + } + + /* No event found, we need to add a new one. */ + el = zmalloc(sizeof(*el)); + el->module = ctx->module; + el->event = event; + el->callback = callback; + listAddNodeTail(RedisModule_EventListeners,el); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ diff --git a/src/redismodule.h b/src/redismodule.h index 19a9cd897..1b8885e04 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -162,6 +162,75 @@ typedef uint64_t RedisModuleTimerID; /* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ #define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) +/* Server events definitions. */ +#define REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED 0 +#define REDISMODULE_EVENT_ID_PERSISTENCE 1 +#define REDISMODULE_EVENT_ID_FLUSHDB 2 +#define REDISMODULE_EVENT_ID_LOADING 3 +#define REDISMODULE_EVENT_ID_CLIENT_CHANGE 4 +#define REDISMODULE_EVENT_ID_SHUTDOWN 5 +#define REDISMODULE_EVENT_ID_REPLICA_CHANGE 6 +#define REDISMODULE_EVENT_ID_MASTER_LINK_CHANGE 7 + +typedef struct RedisModuleEvent { + uint64_t id; /* REDISMODULE_EVENT_ID_... defines. */ + uint64_t dataver; /* Version of the structure we pass as 'data'. */ +} RedisModuleEvent; + +RedisModuleEvent + RedisModuleEvent_ReplicationRoleChanged = { + REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED, + 0 + }, + RedisModuleEvent_Persistence = { + REDISMODULE_EVENT_ID_PERSISTENCE, + 0 + }, + RedisModuleEvent_FlushDB = { + REDISMODULE_EVENT_ID_FLUSHDB, + 0 + }, + RedisModuleEvent_Loading = { + REDISMODULE_EVENT_ID_LOADING, + 0 + }, + RedisModuleEvent_ClientChange = { + REDISMODULE_EVENT_ID_CLIENT_CHANGE, + 0 + }, + RedisModuleEvent_Shutdown = { + REDISMODULE_EVENT_ID_SHUTDOWN, + 0 + }, + RedisModuleEvent_ReplicaChange = { + REDISMODULE_EVENT_ID_REPLICA_CHANGE, + 0 + }, + RedisModuleEvent_MasterLinkChange = { + REDISMODULE_EVENT_ID_MASTER_LINK_CHANGE, + 0 + }; + +typedef int (*RedisModuleEventCallback)(RedisModuleEvent eid, uint64_t subevent, void *data); + +/* Those are values that are used for the 'subevent' callback argument. */ +#define REDISMODULE_EVENT_PERSISTENCE_RDB_START 0 +#define REDISMODULE_EVENT_PERSISTENCE_RDB_END 1 +#define REDISMODULE_EVENT_PERSISTENCE_AOF_START 2 +#define REDISMODULE_EVENT_PERSISTENCE_AOF_END 3 + +#define REDISMODULE_EVENT_LOADING_START 0 +#define REDISMODULE_EVENT_LOADING_END 1 + +#define REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED 0 +#define REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED 1 + +#define REDISMODULE_EVENT_MASTER_LINK_UP 0 +#define REDISMODULE_EVENT_MASTER_LINK_DOWN 1 + +#define REDISMODULE_EVENT_REPLICA_CHANGE_CONNECTED 0 +#define REDISMODULE_EVENT_REPLICA_CHANGE_DISCONNECTED 1 + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE From 35c8b8969e48115fe3197261306a63b2db72c1fc Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 15 Oct 2019 19:31:06 +0200 Subject: [PATCH 189/320] Modules: RM_GetClientInfoById() draft. --- src/module.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++ src/redismodule.h | 17 +++++++++++ 2 files changed, 90 insertions(+) diff --git a/src/module.c b/src/module.c index e3267cdbc..e97a4c1a5 100644 --- a/src/module.c +++ b/src/module.c @@ -1548,6 +1548,79 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { return ctx->client->id; } +/* Return information about the client with the specified ID (that was + * previously obtained via the RedisModule_GetClientId() API). If the + * client exists, REDISMODULE_OK is returned, otherwise REDISMODULE_ERR + * is returned. + * + * When the client exist and the `ci` pointer is not NULL, but points to + * a structure of type RedisModuleClientInfo, previously initialized with + * the correct REDISMODULE_CLIENTINFO_INITIALIZER, the structure is populated + * with the following fields: + * + * uint64_t flags; // REDISMODULE_CLIENTINFO_FLAG_* + * char addr[46]; // IPv4 or IPv6 address. + * uint16_t port; // TCP port. + * uint16_t db; // Selected DB. + * + * With flags having the following meaning: + * + * REDISMODULE_CLIENTINFO_FLAG_SSL Client using SSL connection. + * REDISMODULE_CLIENTINFO_FLAG_PUBSUB Client in Pub/Sub mode. + * REDISMODULE_CLIENTINFO_FLAG_BLOCKED Client blocked in command. + * REDISMODULE_CLIENTINFO_FLAG_TRACKING Client with keys tracking on. + * REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET Client using unix domain socket. + * REDISMODULE_CLIENTINFO_FLAG_MULTI Client in MULTI state. + * + * However passing NULL is a way to just check if the client exists in case + * we are not interested in any additional information. + * + * This is the correct usage when we want the client info structure + * returned: + * + * RedisModuleClientInfo ci = REDISMODULE_CLIENTINFO_INITIALIZER; + * int retval = RedisModule_GetClientInfoById(ctx,&ci); + * if (retval == REDISMODULE_OK) { + * printf("Address: %s\n", ci.addr); + * } + */ +int RM_GetClientInfoById(void *ci, uint64_t id) { + struct { + uint64_t version; /* Version of this structure for ABI compat. */ + uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + char addr[46]; /* IPv4 or IPv6 address. */ + uint16_t port; /* TCP port. */ + uint16_t db; /* Selected DB. */ + } *ci1; + + client *client = lookupClientByID(id); + if (client == NULL) return REDISMODULE_ERR; + if (ci == NULL) return REDISMODULE_OK; + + /* Fill the info structure if passed. */ + uint64_t structver = ((uint64_t*)ci)[0]; + if (structver != 1) return REDISMODULE_ERR; + + ci1 = ci; + memset(ci1,0,sizeof(*ci1)); + if (client->flags & CLIENT_MULTI) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_MULTI; + if (client->flags & CLIENT_PUBSUB) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_PUBSUB; + if (client->flags & CLIENT_UNIX_SOCKET) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET; + if (client->flags & CLIENT_TRACKING) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING; + if (client->flags & CLIENT_BLOCKED) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED; + + int port; + anetPeerToString(client->fd,ci1->addr,sizeof(ci1->addr),&port); + ci1->port = port; + ci1->db = client->db->id; + return REDISMODULE_OK; +} + /* Return the currently selected DB. */ int RM_GetSelectedDb(RedisModuleCtx *ctx) { return ctx->client->db->id; diff --git a/src/redismodule.h b/src/redismodule.h index 1b8885e04..c87e644cb 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -231,6 +231,14 @@ typedef int (*RedisModuleEventCallback)(RedisModuleEvent eid, uint64_t subevent, #define REDISMODULE_EVENT_REPLICA_CHANGE_CONNECTED 0 #define REDISMODULE_EVENT_REPLICA_CHANGE_DISCONNECTED 1 +/* RedisModuleClientInfo flags. */ +#define REDISMODULE_CLIENTINFO_FLAG_SSL (1<<0) +#define REDISMODULE_CLIENTINFO_FLAG_PUBSUB (1<<1) +#define REDISMODULE_CLIENTINFO_FLAG_BLOCKED (1<<2) +#define REDISMODULE_CLIENTINFO_FLAG_TRACKING (1<<3) +#define REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET (1<<4) +#define REDISMODULE_CLIENTINFO_FLAG_MULTI (1<<5) + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE @@ -284,6 +292,15 @@ typedef struct RedisModuleTypeMethods { int aux_save_triggers; } RedisModuleTypeMethods; +#define REDISMODULE_CLIENTINFO_VERSION 1 +typedef struct RedisModuleClientInfo { + uint64_t version; /* Version of this structure for ABI compat. */ + uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + char addr[46]; /* IPv4 or IPv6 address. */ + uint16_t port; /* TCP port. */ + uint16_t db; /* Selected DB. */ +} RedisModuleClientInfo; + #define REDISMODULE_GET_API(name) \ RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name)) From 23ec02d1611e6e5e07731c6e7151d01d9cb478cd Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 Oct 2019 10:15:55 +0200 Subject: [PATCH 190/320] Modules: add the client ID to the client info structure. --- src/module.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/module.c b/src/module.c index e97a4c1a5..3faf99aa6 100644 --- a/src/module.c +++ b/src/module.c @@ -1559,10 +1559,16 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { * with the following fields: * * uint64_t flags; // REDISMODULE_CLIENTINFO_FLAG_* + * uint64_t id; // Client ID * char addr[46]; // IPv4 or IPv6 address. * uint16_t port; // TCP port. * uint16_t db; // Selected DB. * + * Note: the client ID is useless in the context of this call, since we + * already know, however the same structure could be used in other + * contexts where we don't know the client ID, yet the same structure + * is returned. + * * With flags having the following meaning: * * REDISMODULE_CLIENTINFO_FLAG_SSL Client using SSL connection. @@ -1588,6 +1594,7 @@ int RM_GetClientInfoById(void *ci, uint64_t id) { struct { uint64_t version; /* Version of this structure for ABI compat. */ uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + uint64_t id; /* Client ID. */ char addr[46]; /* IPv4 or IPv6 address. */ uint16_t port; /* TCP port. */ uint16_t db; /* Selected DB. */ @@ -1618,6 +1625,7 @@ int RM_GetClientInfoById(void *ci, uint64_t id) { anetPeerToString(client->fd,ci1->addr,sizeof(ci1->addr),&port); ci1->port = port; ci1->db = client->db->id; + ci1->id = client->id; return REDISMODULE_OK; } From 16af0381d589d82f201a8e6d239d3c86acb30a4c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 Oct 2019 10:18:07 +0200 Subject: [PATCH 191/320] Modules: export GetClientInfoById(). --- src/module.c | 1 + src/redismodule.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/module.c b/src/module.c index 3faf99aa6..2507aee19 100644 --- a/src/module.c +++ b/src/module.c @@ -6268,4 +6268,5 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldDouble); REGISTER_API(InfoAddFieldLongLong); REGISTER_API(InfoAddFieldULongLong); + REGISTER_API(GetClientInfoById); } diff --git a/src/redismodule.h b/src/redismodule.h index c87e644cb..8cb1575d4 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -381,6 +381,7 @@ int REDISMODULE_API_FUNC(RedisModule_HashGet)(RedisModuleKey *key, int flags, .. int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos); unsigned long long REDISMODULE_API_FUNC(RedisModule_GetClientId)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_GetClientInfoById)(void *ci, uint64_t id); int REDISMODULE_API_FUNC(RedisModule_GetContextFlags)(RedisModuleCtx *ctx); void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods); @@ -635,6 +636,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldDouble); REDISMODULE_GET_API(InfoAddFieldLongLong); REDISMODULE_GET_API(InfoAddFieldULongLong); + REDISMODULE_GET_API(GetClientInfoById); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); From 482cf667aa6b31e3464d2f202bdac314a9ee1222 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 Oct 2019 10:27:35 +0200 Subject: [PATCH 192/320] Modules: fix RedisModule_GetClientInfoById() example. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 2507aee19..68459b7b4 100644 --- a/src/module.c +++ b/src/module.c @@ -1585,7 +1585,7 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { * returned: * * RedisModuleClientInfo ci = REDISMODULE_CLIENTINFO_INITIALIZER; - * int retval = RedisModule_GetClientInfoById(ctx,&ci); + * int retval = RedisModule_GetClientInfoById(&ci,client_id); * if (retval == REDISMODULE_OK) { * printf("Address: %s\n", ci.addr); * } From e9f6e9306bc51cf454621f5cd91447ffdf2a0d19 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Oct 2019 17:28:24 +0200 Subject: [PATCH 193/320] Modules: refactoring of RM_GetClientInfoById(). --- src/module.c | 70 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/module.c b/src/module.c index 68459b7b4..5711b4c9c 100644 --- a/src/module.c +++ b/src/module.c @@ -1548,6 +1548,45 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { return ctx->client->id; } +/* This is an helper for RM_GetClientInfoById() and other functions: given + * a client, it populates the client info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ci' gets populated. */ +int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + struct { + uint64_t version; /* Version of this structure for ABI compat. */ + uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + uint64_t id; /* Client ID. */ + char addr[46]; /* IPv4 or IPv6 address. */ + uint16_t port; /* TCP port. */ + uint16_t db; /* Selected DB. */ + } *ci1; + + ci1 = ci; + memset(ci1,0,sizeof(*ci1)); + ci1->version = structver; + if (client->flags & CLIENT_MULTI) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_MULTI; + if (client->flags & CLIENT_PUBSUB) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_PUBSUB; + if (client->flags & CLIENT_UNIX_SOCKET) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET; + if (client->flags & CLIENT_TRACKING) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING; + if (client->flags & CLIENT_BLOCKED) + ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED; + + int port; + connPeerToString(client->conn,ci1->addr,sizeof(ci1->addr),&port); + ci1->port = port; + ci1->db = client->db->id; + ci1->id = client->id; + return REDISMODULE_OK; +} + /* Return information about the client with the specified ID (that was * previously obtained via the RedisModule_GetClientId() API). If the * client exists, REDISMODULE_OK is returned, otherwise REDISMODULE_ERR @@ -1591,42 +1630,13 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { * } */ int RM_GetClientInfoById(void *ci, uint64_t id) { - struct { - uint64_t version; /* Version of this structure for ABI compat. */ - uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ - uint64_t id; /* Client ID. */ - char addr[46]; /* IPv4 or IPv6 address. */ - uint16_t port; /* TCP port. */ - uint16_t db; /* Selected DB. */ - } *ci1; - client *client = lookupClientByID(id); if (client == NULL) return REDISMODULE_ERR; if (ci == NULL) return REDISMODULE_OK; /* Fill the info structure if passed. */ uint64_t structver = ((uint64_t*)ci)[0]; - if (structver != 1) return REDISMODULE_ERR; - - ci1 = ci; - memset(ci1,0,sizeof(*ci1)); - if (client->flags & CLIENT_MULTI) - ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_MULTI; - if (client->flags & CLIENT_PUBSUB) - ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_PUBSUB; - if (client->flags & CLIENT_UNIX_SOCKET) - ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET; - if (client->flags & CLIENT_TRACKING) - ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_TRACKING; - if (client->flags & CLIENT_BLOCKED) - ci1->flags |= REDISMODULE_CLIENTINFO_FLAG_BLOCKED; - - int port; - anetPeerToString(client->fd,ci1->addr,sizeof(ci1->addr),&port); - ci1->port = port; - ci1->db = client->db->id; - ci1->id = client->id; - return REDISMODULE_OK; + return modulePopulateClientInfoStructure(ci,client,structver); } /* Return the currently selected DB. */ From aa9b870db3ccf823dfe4488d1e2114dbab1dbf1f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Oct 2019 17:51:18 +0200 Subject: [PATCH 194/320] Modules hooks: event firing logic. --- src/module.c | 89 +++++++++++++++++++++++++++++++++++++++++------ src/redismodule.h | 18 +++++----- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/src/module.c b/src/module.c index 5711b4c9c..6de56a048 100644 --- a/src/module.c +++ b/src/module.c @@ -1553,19 +1553,22 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { * fields depending on the version provided. If the version is not valid * then REDISMODULE_ERR is returned. Otherwise the function returns * REDISMODULE_OK and the structure pointed by 'ci' gets populated. */ + +/* Note that we may have multiple versions of the client info structure, + * as the API evolves. */ +struct moduleClientInfoV1 { + uint64_t version; /* Version of this structure for ABI compat. */ + uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + uint64_t id; /* Client ID. */ + char addr[46]; /* IPv4 or IPv6 address. */ + uint16_t port; /* TCP port. */ + uint16_t db; /* Selected DB. */ +}; + int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { if (structver != 1) return REDISMODULE_ERR; - struct { - uint64_t version; /* Version of this structure for ABI compat. */ - uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ - uint64_t id; /* Client ID. */ - char addr[46]; /* IPv4 or IPv6 address. */ - uint16_t port; /* TCP port. */ - uint16_t db; /* Selected DB. */ - } *ci1; - - ci1 = ci; + struct moduleClientInfoV1 *ci1 = ci; memset(ci1,0,sizeof(*ci1)); ci1->version = structver; if (client->flags & CLIENT_MULTI) @@ -5685,6 +5688,40 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * will be unsubscribed. If there was a previous subscription and the callback * is not null, the old callback will be replaced with the new one. * + * The callback must be of this type: + * + * int (*RedisModuleEventCallback)(RedisModuleCtx *ctx, + * RedisModuleEvent eid, + * uint64_t subevent, + * void *data); + * + * The 'ctx' is a normal Redis module context that the callback can use in + * order to call other modules APIs. The 'eid' is the event itself, this + * is only useful in the case the module subscribed to multiple events: using + * the 'id' field of this structure it is possible to check if the event + * is one of the events we registered with this callback. The 'subevent' field + * depends on the event that fired. Here is a list of sub events: + * + * REDISMODULE_EVENT_PERSISTENCE_RDB_START + * REDISMODULE_EVENT_PERSISTENCE_RDB_END + * REDISMODULE_EVENT_PERSISTENCE_AOF_START + * REDISMODULE_EVENT_PERSISTENCE_AOF_END + * REDISMODULE_EVENT_LOADING_START + * REDISMODULE_EVENT_LOADING_END + * REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED + * REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED + * REDISMODULE_EVENT_MASTER_LINK_UP + * REDISMODULE_EVENT_MASTER_LINK_DOWN + * REDISMODULE_EVENT_REPLICA_CHANGE_CONNECTED + * REDISMODULE_EVENT_REPLICA_CHANGE_DISCONNECTED + * + * Finally the 'data' pointer may be populated, only for certain events, with + * more relevant data. + * + * In the case of REDISMODULE_EVENT_CLIENT_CHANGE events it gets populated + * with a RedisModuleClientInfo structure: the module should just cast the + * void pointer to the structure and access its fields. + * * The function returns REDISMODULE_OK if the module was successfully subscrived * for the specified event. If the API is called from a wrong context then * REDISMODULE_ERR is returned. */ @@ -5722,6 +5759,38 @@ int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent eve return REDISMODULE_OK; } +/* This is called by the Redis internals every time we want to fire an + * event that can be interceppted by some module. The pointer 'data' is useful + * in order to populate the event-specific structure when needed, in order + * to return the structure with more information to the callback. + * + * 'eid' and 'subid' are just the main event ID and the sub event associated + * with the event, depending on what exactly happened. */ +void RedisModule_FireServerEvent(uint64_t eid, int subid, void *data) { + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + while((ln = listNext(&li))) { + RedisModuleEventListener *el = ln->value; + if (el->event.id == eid) { + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = el->module; + ctx.client = moduleFreeContextReusedClient; + + void *moduledata = NULL; + struct moduleClientInfoV1 civ1; + if (eid == REDISMODULE_EVENT_ID_CLIENT_CONNNECTED || + eid == REDISMODULE_EVENT_ID_CLIENT_DISCONNECTED) + { + modulePopulateClientInfoStructure(&civ1,data, + el->event.dataver); + } + el->callback(&ctx,el->event,subid,moduledata); + moduleFreeContext(&ctx); + } + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ diff --git a/src/redismodule.h b/src/redismodule.h index 8cb1575d4..d7ada3d5b 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -180,38 +180,38 @@ typedef struct RedisModuleEvent { RedisModuleEvent RedisModuleEvent_ReplicationRoleChanged = { REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED, - 0 + 1 }, RedisModuleEvent_Persistence = { REDISMODULE_EVENT_ID_PERSISTENCE, - 0 + 1 }, RedisModuleEvent_FlushDB = { REDISMODULE_EVENT_ID_FLUSHDB, - 0 + 1 }, RedisModuleEvent_Loading = { REDISMODULE_EVENT_ID_LOADING, - 0 + 1 }, RedisModuleEvent_ClientChange = { REDISMODULE_EVENT_ID_CLIENT_CHANGE, - 0 + 1 }, RedisModuleEvent_Shutdown = { REDISMODULE_EVENT_ID_SHUTDOWN, - 0 + 1 }, RedisModuleEvent_ReplicaChange = { REDISMODULE_EVENT_ID_REPLICA_CHANGE, - 0 + 1 }, RedisModuleEvent_MasterLinkChange = { REDISMODULE_EVENT_ID_MASTER_LINK_CHANGE, - 0 + 1 }; -typedef int (*RedisModuleEventCallback)(RedisModuleEvent eid, uint64_t subevent, void *data); +typedef int (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); /* Those are values that are used for the 'subevent' callback argument. */ #define REDISMODULE_EVENT_PERSISTENCE_RDB_START 0 From 30d8dce935ad96c493993709361c2ac54b4cf3e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 10:15:12 +0200 Subject: [PATCH 195/320] Modules hooks: fix a leak and a few more issues. --- src/module.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/module.c b/src/module.c index 6de56a048..0d96e3391 100644 --- a/src/module.c +++ b/src/module.c @@ -328,23 +328,6 @@ static struct RedisModuleForkInfo { * the start and end of an RDB or AOF save, the change of role in replication, * and similar other events. */ -#define REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED 0 -#define REDISMODULE_EVENT_ID_RDB_SAVE_START 1 -#define REDISMODULE_EVENT_ID_RDB_SAVE_END 2 -#define REDISMODULE_EVENT_ID_AOF_REWRITE_START 3 -#define REDISMODULE_EVENT_ID_AOF_REWRITE_END 4 -#define REDISMODULE_EVENT_ID_FLUSHDB_START 5 -#define REDISMODULE_EVENT_ID_FLUSHDB_END 6 -#define REDISMODULE_EVENT_ID_LOADING_START 7 -#define REDISMODULE_EVENT_ID_LOADING_END 8 -#define REDISMODULE_EVENT_ID_CLIENT_CONNNECTED 9 -#define REDISMODULE_EVENT_ID_CLIENT_DISCONNECTED 10 -#define REDISMODULE_EVENT_ID_SERVER_SHUTDOWN 11 -#define REDISMODULE_EVENT_ID_REPLICA_ONLINE 12 -#define REDISMODULE_EVENT_ID_REPLICA_OFFLINE 13 -#define REDISMODULE_EVENT_ID_MASTER_LINK_UP 14 -#define REDISMODULE_EVENT_ID_MASTER_LINK_DOWN 15 - typedef struct RedisModuleEventListener { RedisModule *module; RedisModuleEvent event; @@ -5743,10 +5726,12 @@ int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent eve /* Modify or remove the event listener if we already had one. */ if (ln) { - if (callback == NULL) + if (callback == NULL) { listDelNode(RedisModule_EventListeners,ln); - else + zfree(el); + } else { el->callback = callback; /* Update the callback with the new one. */ + } return REDISMODULE_OK; } @@ -5767,6 +5752,11 @@ int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent eve * 'eid' and 'subid' are just the main event ID and the sub event associated * with the event, depending on what exactly happened. */ void RedisModule_FireServerEvent(uint64_t eid, int subid, void *data) { + /* Fast path to return ASAP if there is nothing to do, avoiding to + * setup the iterator and so forth: we want this call to be extremely + * cheap if there are no registered modules. */ + if (listLength(RedisModule_EventListeners) == 0) return; + listIter li; listNode *ln; listRewind(RedisModule_EventListeners,&li); @@ -5779,9 +5769,7 @@ void RedisModule_FireServerEvent(uint64_t eid, int subid, void *data) { void *moduledata = NULL; struct moduleClientInfoV1 civ1; - if (eid == REDISMODULE_EVENT_ID_CLIENT_CONNNECTED || - eid == REDISMODULE_EVENT_ID_CLIENT_DISCONNECTED) - { + if (eid == REDISMODULE_EVENT_ID_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); } From 98a0d95965944f3cd90602fcf51b02bb419705c7 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 10:24:50 +0200 Subject: [PATCH 196/320] Modules hooks: handle module unloading + API export. --- src/module.c | 27 ++++++++++++++++++++++----- src/redismodule.h | 2 ++ src/server.h | 1 + 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 0d96e3391..0cfc92416 100644 --- a/src/module.c +++ b/src/module.c @@ -5708,7 +5708,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * The function returns REDISMODULE_OK if the module was successfully subscrived * for the specified event. If the API is called from a wrong context then * REDISMODULE_ERR is returned. */ -int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) { +int RM_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback) { RedisModuleEventListener *el; /* Protect in case of calls from contexts without a module reference. */ @@ -5751,7 +5751,7 @@ int RedisModule_SubscribeToServerEvent(RedisModuleCtx *ctx, RedisModuleEvent eve * * 'eid' and 'subid' are just the main event ID and the sub event associated * with the event, depending on what exactly happened. */ -void RedisModule_FireServerEvent(uint64_t eid, int subid, void *data) { +void moduleFireServerEvent(uint64_t eid, int subid, void *data) { /* Fast path to return ASAP if there is nothing to do, avoiding to * setup the iterator and so forth: we want this call to be extremely * cheap if there are no registered modules. */ @@ -5779,6 +5779,23 @@ void RedisModule_FireServerEvent(uint64_t eid, int subid, void *data) { } } +/* Remove all the listeners for this module: this is used before unloading + * a module. */ +void moduleUnsubscribeAllServerEvents(RedisModule *module) { + RedisModuleEventListener *el; + listIter li; + listNode *ln; + listRewind(RedisModule_EventListeners,&li); + + while((ln = listNext(&li))) { + el = ln->value; + if (el->module == module) { + listDelNode(RedisModule_EventListeners,ln); + zfree(el); + } + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -5970,7 +5987,7 @@ int moduleUnload(sds name) { errno = EPERM; return REDISMODULE_ERR; } - + /* Give module a chance to clean up. */ int (*onunload)(void *); onunload = (int (*)(void *))(unsigned long) dlsym(module->handle, "RedisModule_OnUnload"); @@ -5995,8 +6012,7 @@ int moduleUnload(sds name) { /* Remove any notification subscribers this module might have */ moduleUnsubscribeNotifications(module); - - /* Unregister all the hooks. TODO: Yet no hooks support here. */ + moduleUnsubscribeAllServerEvents(module); /* Unload the dynamic library. */ if (dlclose(module->handle) == -1) { @@ -6336,4 +6352,5 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldLongLong); REGISTER_API(InfoAddFieldULongLong); REGISTER_API(GetClientInfoById); + REGISTER_API(SubscribeToServerEvent); } diff --git a/src/redismodule.h b/src/redismodule.h index d7ada3d5b..0bc6d2c3e 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -446,6 +446,7 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ct int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); +int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -637,6 +638,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldLongLong); REDISMODULE_GET_API(InfoAddFieldULongLong); REDISMODULE_GET_API(GetClientInfoById); + REDISMODULE_GET_API(SubscribeToServerEvent); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/src/server.h b/src/server.h index 829d8213e..a427ef373 100644 --- a/src/server.h +++ b/src/server.h @@ -1598,6 +1598,7 @@ int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); +void moduleFireServerEvent(uint64_t eid, int subid, void *data); /* Utils */ long long ustime(void); From f3e4f24f68225f65ca7d72c719245bc8037c9611 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 10:44:18 +0200 Subject: [PATCH 197/320] Modules hooks: fix define / linker issues. Implement one test event. --- src/module.c | 5 +--- src/networking.c | 4 ++- src/redismodule.h | 67 +++++++++++++++++++++++++---------------------- src/server.h | 3 +++ 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/src/module.c b/src/module.c index 0cfc92416..63d2ac1bb 100644 --- a/src/module.c +++ b/src/module.c @@ -33,9 +33,6 @@ #include #include -#define REDISMODULE_CORE 1 -#include "redismodule.h" - /* -------------------------------------------------------------------------- * Private data structures used by the modules system. Those are data * structures that are never exposed to Redis Modules, if not as void @@ -5769,7 +5766,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { void *moduledata = NULL; struct moduleClientInfoV1 civ1; - if (eid == REDISMODULE_EVENT_ID_CLIENT_CHANGE) { + if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); } diff --git a/src/networking.c b/src/networking.c index ddfe4d8e3..c20c13e9a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -827,9 +827,11 @@ void clientAcceptHandler(connection *conn) { } server.stat_numconnections++; + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + c); } - #define MAX_ACCEPTS_PER_CALL 1000 static void acceptCommonHandler(connection *conn, int flags, char *ip) { client *c; diff --git a/src/redismodule.h b/src/redismodule.h index 0bc6d2c3e..02cfa9586 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -163,73 +163,76 @@ typedef uint64_t RedisModuleTimerID; #define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) /* Server events definitions. */ -#define REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED 0 -#define REDISMODULE_EVENT_ID_PERSISTENCE 1 -#define REDISMODULE_EVENT_ID_FLUSHDB 2 -#define REDISMODULE_EVENT_ID_LOADING 3 -#define REDISMODULE_EVENT_ID_CLIENT_CHANGE 4 -#define REDISMODULE_EVENT_ID_SHUTDOWN 5 -#define REDISMODULE_EVENT_ID_REPLICA_CHANGE 6 -#define REDISMODULE_EVENT_ID_MASTER_LINK_CHANGE 7 +#define REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED 0 +#define REDISMODULE_EVENT_PERSISTENCE 1 +#define REDISMODULE_EVENT_FLUSHDB 2 +#define REDISMODULE_EVENT_LOADING 3 +#define REDISMODULE_EVENT_CLIENT_CHANGE 4 +#define REDISMODULE_EVENT_SHUTDOWN 5 +#define REDISMODULE_EVENT_REPLICA_CHANGE 6 +#define REDISMODULE_EVENT_MASTER_LINK_CHANGE 7 typedef struct RedisModuleEvent { - uint64_t id; /* REDISMODULE_EVENT_ID_... defines. */ + uint64_t id; /* REDISMODULE_EVENT_... defines. */ uint64_t dataver; /* Version of the structure we pass as 'data'. */ } RedisModuleEvent; -RedisModuleEvent +struct RedisModuleCtx; +typedef int (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); + +static RedisModuleEvent RedisModuleEvent_ReplicationRoleChanged = { - REDISMODULE_EVENT_ID_REPLICATION_ROLE_CHANGED, + REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, 1 }, RedisModuleEvent_Persistence = { - REDISMODULE_EVENT_ID_PERSISTENCE, + REDISMODULE_EVENT_PERSISTENCE, 1 }, RedisModuleEvent_FlushDB = { - REDISMODULE_EVENT_ID_FLUSHDB, + REDISMODULE_EVENT_FLUSHDB, 1 }, RedisModuleEvent_Loading = { - REDISMODULE_EVENT_ID_LOADING, + REDISMODULE_EVENT_LOADING, 1 }, RedisModuleEvent_ClientChange = { - REDISMODULE_EVENT_ID_CLIENT_CHANGE, + REDISMODULE_EVENT_CLIENT_CHANGE, 1 }, RedisModuleEvent_Shutdown = { - REDISMODULE_EVENT_ID_SHUTDOWN, + REDISMODULE_EVENT_SHUTDOWN, 1 }, RedisModuleEvent_ReplicaChange = { - REDISMODULE_EVENT_ID_REPLICA_CHANGE, + REDISMODULE_EVENT_REPLICA_CHANGE, 1 }, RedisModuleEvent_MasterLinkChange = { - REDISMODULE_EVENT_ID_MASTER_LINK_CHANGE, + REDISMODULE_EVENT_MASTER_LINK_CHANGE, 1 }; -typedef int (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); - /* Those are values that are used for the 'subevent' callback argument. */ -#define REDISMODULE_EVENT_PERSISTENCE_RDB_START 0 -#define REDISMODULE_EVENT_PERSISTENCE_RDB_END 1 -#define REDISMODULE_EVENT_PERSISTENCE_AOF_START 2 -#define REDISMODULE_EVENT_PERSISTENCE_AOF_END 3 +#define REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START 0 +#define REDISMODULE_SUBEVENT_PERSISTENCE_RDB_END 1 +#define REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START 2 +#define REDISMODULE_SUBEVENT_PERSISTENCE_AOF_END 3 -#define REDISMODULE_EVENT_LOADING_START 0 -#define REDISMODULE_EVENT_LOADING_END 1 +#define REDISMODULE_SUBEVENT_LOADING_RDB_START 0 +#define REDISMODULE_SUBEVENT_LOADING_RDB_END 1 +#define REDISMODULE_SUBEVENT_LOADING_AOF_START 2 +#define REDISMODULE_SUBEVENT_LOADING_AOF_END 3 -#define REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED 0 -#define REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED 1 +#define REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED 0 +#define REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED 1 -#define REDISMODULE_EVENT_MASTER_LINK_UP 0 -#define REDISMODULE_EVENT_MASTER_LINK_DOWN 1 +#define REDISMODULE_SUBEVENT_MASTER_LINK_UP 0 +#define REDISMODULE_SUBEVENT_MASTER_LINK_DOWN 1 -#define REDISMODULE_EVENT_REPLICA_CHANGE_CONNECTED 0 -#define REDISMODULE_EVENT_REPLICA_CHANGE_DISCONNECTED 1 +#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_CONNECTED 0 +#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_DISCONNECTED 1 /* RedisModuleClientInfo flags. */ #define REDISMODULE_CLIENTINFO_FLAG_SSL (1<<0) diff --git a/src/server.h b/src/server.h index a427ef373..97672d727 100644 --- a/src/server.h +++ b/src/server.h @@ -68,6 +68,9 @@ typedef long long mstime_t; /* millisecond time type. */ #include "rax.h" /* Radix tree */ #include "connection.h" /* Connection abstraction */ +#define REDISMODULE_CORE 1 +#include "redismodule.h" /* Redis modules API defines. */ + /* Following includes allow test functions to be called from Redis main() */ #include "zipmap.h" #include "sha1.h" From d5bd45a368ca6bc2888d9c28a4d1d61ebfcad771 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 12:48:07 +0200 Subject: [PATCH 198/320] Modules hooks: fix sub event in calls from client change. --- src/networking.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index c20c13e9a..dec49642f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -828,7 +828,7 @@ void clientAcceptHandler(connection *conn) { server.stat_numconnections++; moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, - REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED, c); } @@ -1043,6 +1043,9 @@ void freeClient(client *c) { freeClientAsync(c); return; } + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + c); /* If it is our master that's beging disconnected we should make sure * to cache the state to try a partial resynchronization later. From 7379bb76f29b3aea3c13e29b0f7d26fdc54345c8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 13:26:29 +0200 Subject: [PATCH 199/320] Module hooks: fixes and an example module. --- src/module.c | 1 + src/modules/Makefile | 8 +++++- src/modules/hellohook.c | 60 +++++++++++++++++++++++++++++++++++++++++ src/redismodule.h | 3 ++- 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 src/modules/hellohook.c diff --git a/src/module.c b/src/module.c index 63d2ac1bb..c1c7cf1a8 100644 --- a/src/module.c +++ b/src/module.c @@ -5769,6 +5769,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); + moduledata = &civ1; } el->callback(&ctx,el->event,subid,moduledata); moduleFreeContext(&ctx); diff --git a/src/modules/Makefile b/src/modules/Makefile index 4f6b50f2e..9fb84cff8 100644 --- a/src/modules/Makefile +++ b/src/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so +all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellohook.so .c.xo: $(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -46,6 +46,12 @@ hellotimer.so: hellotimer.xo hellodict.xo: ../redismodule.h hellodict.so: hellodict.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +hellohook.xo: ../redismodule.h + +hellohook.so: hellohook.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc testmodule.xo: ../redismodule.h diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c new file mode 100644 index 000000000..ec3c9d264 --- /dev/null +++ b/src/modules/hellohook.c @@ -0,0 +1,60 @@ +/* Server hooks API example + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "../redismodule.h" +#include +#include +#include +#include + +/* Client state change callback. */ +void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + RedisModuleClientInfo *ci = data; + printf("Client event for client #%llu %s:%d\n",ci->id,ci->addr,ci->port); +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"hellohook",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ClientChange, clientChangeCallback); + return REDISMODULE_OK; +} diff --git a/src/redismodule.h b/src/redismodule.h index 02cfa9586..9f7cc7e72 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -178,7 +178,7 @@ typedef struct RedisModuleEvent { } RedisModuleEvent; struct RedisModuleCtx; -typedef int (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); +typedef void (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); static RedisModuleEvent RedisModuleEvent_ReplicationRoleChanged = { @@ -299,6 +299,7 @@ typedef struct RedisModuleTypeMethods { typedef struct RedisModuleClientInfo { uint64_t version; /* Version of this structure for ABI compat. */ uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + uint64_t id; /* Client ID. */ char addr[46]; /* IPv4 or IPv6 address. */ uint16_t port; /* TCP port. */ uint16_t db; /* Selected DB. */ From aa4d69606d585fbff7e721a6c48db3d156cd12d1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 16:50:52 +0200 Subject: [PATCH 200/320] Modules hooks: do more in example client callback. --- src/modules/hellohook.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c index ec3c9d264..ea0ac517d 100644 --- a/src/modules/hellohook.c +++ b/src/modules/hellohook.c @@ -41,8 +41,13 @@ void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) { REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + RedisModuleClientInfo *ci = data; - printf("Client event for client #%llu %s:%d\n",ci->id,ci->addr,ci->port); + printf("Client %s event for client #%llu %s:%d\n", + (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? + "connection" : "disconnection", + ci->id,ci->addr,ci->port); } /* This function must be present on each Redis module. It is used in order to From d41d09d474925f242faabb7a2f3f937030040edb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 17:15:22 +0200 Subject: [PATCH 201/320] Modules hooks: document what yet to implement WIP 1. --- src/module.c | 80 ++++++++++++++++++++++++++++++++++++++++++----- src/redismodule.h | 5 +++ 2 files changed, 78 insertions(+), 7 deletions(-) diff --git a/src/module.c b/src/module.c index c1c7cf1a8..8f6c1d890 100644 --- a/src/module.c +++ b/src/module.c @@ -5680,14 +5680,80 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * is only useful in the case the module subscribed to multiple events: using * the 'id' field of this structure it is possible to check if the event * is one of the events we registered with this callback. The 'subevent' field - * depends on the event that fired. Here is a list of sub events: + * depends on the event that fired. + * + * Here is a list of events you can use as 'eid' and related sub events: + * + * RedisModuleEvent_ReplicationRoleChanged + * + * This event is called when the instance switches from master + * to replica or the other way around, however the event is + * also called when the replica remains a replica but starts to + * replicate with a different master. + * + * The following sub events are available: + * + * REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER + * REDISMODULE_EVENT_REPLROLECHANGED_NOW_REPLICA + * + * The 'data' field can be casted by the callback to a + * RedisModuleReplicationInfo structure with the following fields: + * + * int master; // true if master, false if replica + * char *masterhost; // master instance hostname for NOW_REPLICA + * int masterport; // master instance port for NOW_REPLICA + * char *replid1; // Main replication ID + * char *replid2; // Secondary replication ID + * uint64_t repl2_offset; // Offset of replid2 validity + * uint64_t main_repl_offset; // Replication offset + * + * RedisModuleEvent_Persistence + * + * This event is called when RDB saving or AOF rewriting starts + * and ends. The following sub events are available: + * + * REDISMODULE_EVENT_LOADING_RDB_START // BGSAVE start + * REDISMODULE_EVENT_LOADING_RDB_END // BGSAVE end + * REDISMODULE_EVENT_LOADING_SYNC_RDB_START // SAVE start + * REDISMODULE_EVENT_LOADING_SYNC_RDB_START // SAVE end + * REDISMODULE_EVENT_LOADING_AOF_START // AOF rewrite start + * REDISMODULE_EVENT_LOADING_AOF_END // AOF rewrite end + * + * The above events are triggered not just when the user calls the + * relevant commands like BGSAVE, but also when a saving operation + * or AOF rewriting occurs because of internal server triggers. + * + * RedisModuleEvent_FlushDB + * + * The FLUSHALL, FLUSHDB or an internal flush (for instance + * because of replication, after the replica synchronization) + * happened. The following sub events are available: + * + * REDISMODULE_EVENT_FLUSHALL_START + * REDISMODULE_EVENT_FLUSHALL_END + * + * The data pointer can be casted to a RedisModuleFlushInfo + * structure with the following fields: + * + * int async; // True if the flush is done in a thread. + * See for instance FLUSHALL ASYNC. + * In this case the END callback is invoked + * immediately after the database is put + * in the free list of the thread. + * int dbnum; // Flushed database number, -1 for all the DBs + * in the case of the FLUSHALL operation. + * + * The start event is called *before* the operation is initated, thus + * allowing the callback to call DBSIZE or other operation on the + * yet-to-free keyspace. + * + * RedisModuleEvent_Loading + * RedisModuleEvent_ClientChange + * RedisModuleEvent_Shutdown + * RedisModuleEvent_ReplicaChange + * RedisModuleEvent_CronLoop + * RedisModuleEvent_MasterLinkChange * - * REDISMODULE_EVENT_PERSISTENCE_RDB_START - * REDISMODULE_EVENT_PERSISTENCE_RDB_END - * REDISMODULE_EVENT_PERSISTENCE_AOF_START - * REDISMODULE_EVENT_PERSISTENCE_AOF_END - * REDISMODULE_EVENT_LOADING_START - * REDISMODULE_EVENT_LOADING_END * REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED * REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED * REDISMODULE_EVENT_MASTER_LINK_UP diff --git a/src/redismodule.h b/src/redismodule.h index 9f7cc7e72..2a413a3c8 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -171,6 +171,7 @@ typedef uint64_t RedisModuleTimerID; #define REDISMODULE_EVENT_SHUTDOWN 5 #define REDISMODULE_EVENT_REPLICA_CHANGE 6 #define REDISMODULE_EVENT_MASTER_LINK_CHANGE 7 +#define REDISMODULE_EVENT_CRON_LOOP 8 typedef struct RedisModuleEvent { uint64_t id; /* REDISMODULE_EVENT_... defines. */ @@ -209,6 +210,10 @@ static RedisModuleEvent REDISMODULE_EVENT_REPLICA_CHANGE, 1 }, + RedisModuleEvent_CronLoop = { + REDISMODULE_EVENT_CRON_LOOP, + 1 + }, RedisModuleEvent_MasterLinkChange = { REDISMODULE_EVENT_MASTER_LINK_CHANGE, 1 From 4866552961109a362ab49cb8bb2ed883d5d40d02 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 Oct 2019 17:32:39 +0200 Subject: [PATCH 202/320] Modules hooks: document what yet to implement WIP 2. --- src/module.c | 75 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 15 deletions(-) diff --git a/src/module.c b/src/module.c index 8f6c1d890..37d89fdfe 100644 --- a/src/module.c +++ b/src/module.c @@ -5682,6 +5682,9 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * is one of the events we registered with this callback. The 'subevent' field * depends on the event that fired. * + * Finally the 'data' pointer may be populated, only for certain events, with + * more relevant data. + * * Here is a list of events you can use as 'eid' and related sub events: * * RedisModuleEvent_ReplicationRoleChanged @@ -5747,26 +5750,68 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * allowing the callback to call DBSIZE or other operation on the * yet-to-free keyspace. * - * RedisModuleEvent_Loading - * RedisModuleEvent_ClientChange - * RedisModuleEvent_Shutdown + * RedisModuleEvent_Loading + * + * Called on loading operations: at startup when the server is + * started, but also after a first synchronization when the + * replica is loading the RDB file from the master. + * The following sub events are available: + * + * REDISMODULE_EVENT_LOADING_RDB_START + * REDISMODULE_EVENT_LOADING_RDB_END + * REDISMODULE_EVENT_LOADING_MASTER_RDB_START + * REDISMODULE_EVENT_LOADING_MASTER_RDB_END + * REDISMODULE_EVENT_LOADING_AOF_START + * REDISMODULE_EVENT_LOADING_AOF_END + * + * RedisModuleEvent_ClientChange + * + * Called when a client connects or disconnects. + * The data pointer can be casted to a RedisModuleClientInfo + * structure, documented in RedisModule_GetClientInfoById(). + * The following sub events are available: + * + * REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED + * REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED + * + * RedisModuleEvent_Shutdown + * + * The server is shutting down. No subevents are available. + * * RedisModuleEvent_ReplicaChange + * + * This event is called when the instance (that can be both a + * master or a replica) get a new online replica, or lose a + * replica since it gets disconnected. + * The following sub events are availble: + * + * REDISMODULE_EVENT_REPLICA_CHANGE_ONLINE + * REDISMODULE_EVENT_REPLICA_CHANGE_OFFLINE + * + * No additional information is available so far: future versions + * of Redis will have an API in order to enumerate the replicas + * connected and their state. + * * RedisModuleEvent_CronLoop + * + * This event is called every time Redis calls the serverCron() + * function in order to do certain bookkeeping. Modules that are + * required to do operations from time to time may use this callback. + * Normally Redis calls this function 10 times per second, but + * this changes depending on the "hz" configuration. + * No sub events are available. + * * RedisModuleEvent_MasterLinkChange * - * REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED - * REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED - * REDISMODULE_EVENT_MASTER_LINK_UP - * REDISMODULE_EVENT_MASTER_LINK_DOWN - * REDISMODULE_EVENT_REPLICA_CHANGE_CONNECTED - * REDISMODULE_EVENT_REPLICA_CHANGE_DISCONNECTED + * This is called for replicas in order to notify when the + * replication link becomes functional (up) with our master, + * or when it goes down. Note that the link is not considered + * up when we just connected to the master, but only if the + * replication is happening correctly. + * The following sub events are available: * - * Finally the 'data' pointer may be populated, only for certain events, with - * more relevant data. - * - * In the case of REDISMODULE_EVENT_CLIENT_CHANGE events it gets populated - * with a RedisModuleClientInfo structure: the module should just cast the - * void pointer to the structure and access its fields. + * REDISMODULE_EVENT_MASTER_LINK_UP + * REDISMODULE_EVENT_MASTER_LINK_DOWN * * The function returns REDISMODULE_OK if the module was successfully subscrived * for the specified event. If the API is called from a wrong context then From 6cd26b3db3f456f37653d11fd4bfe082e4d81ddb Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:14:23 +0200 Subject: [PATCH 203/320] Modules hooks: unify structures definitions. --- src/module.c | 19 ++++--------------- src/redismodule.h | 48 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/module.c b/src/module.c index 37d89fdfe..8957849fd 100644 --- a/src/module.c +++ b/src/module.c @@ -1534,21 +1534,10 @@ unsigned long long RM_GetClientId(RedisModuleCtx *ctx) { * then REDISMODULE_ERR is returned. Otherwise the function returns * REDISMODULE_OK and the structure pointed by 'ci' gets populated. */ -/* Note that we may have multiple versions of the client info structure, - * as the API evolves. */ -struct moduleClientInfoV1 { - uint64_t version; /* Version of this structure for ABI compat. */ - uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ - uint64_t id; /* Client ID. */ - char addr[46]; /* IPv4 or IPv6 address. */ - uint16_t port; /* TCP port. */ - uint16_t db; /* Selected DB. */ -}; - int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { if (structver != 1) return REDISMODULE_ERR; - struct moduleClientInfoV1 *ci1 = ci; + RedisModuleClientInfoV1 *ci1 = ci; memset(ci1,0,sizeof(*ci1)); ci1->version = structver; if (client->flags & CLIENT_MULTI) @@ -5738,12 +5727,12 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * The data pointer can be casted to a RedisModuleFlushInfo * structure with the following fields: * - * int async; // True if the flush is done in a thread. + * int32_t async; // True if the flush is done in a thread. * See for instance FLUSHALL ASYNC. * In this case the END callback is invoked * immediately after the database is put * in the free list of the thread. - * int dbnum; // Flushed database number, -1 for all the DBs + * int32_t dbnum; // Flushed database number, -1 for all the DBs * in the case of the FLUSHALL operation. * * The start event is called *before* the operation is initated, thus @@ -5876,7 +5865,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { ctx.client = moduleFreeContextReusedClient; void *moduledata = NULL; - struct moduleClientInfoV1 civ1; + RedisModuleClientInfoV1 civ1; if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); diff --git a/src/redismodule.h b/src/redismodule.h index 2a413a3c8..d8f4fb901 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -247,6 +247,44 @@ static RedisModuleEvent #define REDISMODULE_CLIENTINFO_FLAG_UNIXSOCKET (1<<4) #define REDISMODULE_CLIENTINFO_FLAG_MULTI (1<<5) +/* Here we take all the structures that the module pass to the core + * and the other way around. Notably the list here contains the structures + * used by the hooks API RedisModule_RegisterToServerEvent(). + * + * The structures always start with a 'version' field. This is useful + * when we want to pass a reference to the structure to the core APIs, + * for the APIs to fill the structure. In that case, the structure 'version' + * field is initialized before passing it to the core, so that the core is + * able to cast the pointer to the appropriate structure version. In this + * way we obtain ABI compatibility. + * + * Here we'll list all the structure versions in case they evolve over time, + * however using a define, we'll make sure to use the last version as the + * public name for the module to use. */ + +#define REDISMODULE_CLIENTINFO_VERSION 1 +typedef struct RedisModuleClientInfo { + uint64_t version; /* Version of this structure for ABI compat. */ + uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ + uint64_t id; /* Client ID. */ + char addr[46]; /* IPv4 or IPv6 address. */ + uint16_t port; /* TCP port. */ + uint16_t db; /* Selected DB. */ +} RedisModuleClientInfoV1; + +#define RedisModuleClientInfo RedisModuleClientInfoV1 + +#define REDISMODULE_FLUSHINFO_VERSION 1 +typedef struct RedisModuleFlushInfo { + uint64_t version; /* Not used since this structure is never passed + from the module to the core right now. Here + for future compatibility. */ + int32_t sync; /* Synchronous or threaded flush?. */ + int32_t dbnum; /* Flushed database number, -1 for ALL. */ +} RedisModuleFlushInfoV1; + +#define RedisModuleFlushInfo RedisModuleFlushInfoV1 + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE @@ -300,16 +338,6 @@ typedef struct RedisModuleTypeMethods { int aux_save_triggers; } RedisModuleTypeMethods; -#define REDISMODULE_CLIENTINFO_VERSION 1 -typedef struct RedisModuleClientInfo { - uint64_t version; /* Version of this structure for ABI compat. */ - uint64_t flags; /* REDISMODULE_CLIENTINFO_FLAG_* */ - uint64_t id; /* Client ID. */ - char addr[46]; /* IPv4 or IPv6 address. */ - uint16_t port; /* TCP port. */ - uint16_t db; /* Selected DB. */ -} RedisModuleClientInfo; - #define REDISMODULE_GET_API(name) \ RedisModule_GetApi("RedisModule_" #name, ((void **)&RedisModule_ ## name)) From 581c9133e591226d885e8d0d76dc616a54bf8b68 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:22:46 +0200 Subject: [PATCH 204/320] Modules hooks: implement the FLUSHDB event. --- src/db.c | 13 +++++++++++++ src/module.c | 4 ++-- src/redismodule.h | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index f7d3b71e8..2c0a0cdd3 100644 --- a/src/db.c +++ b/src/db.c @@ -350,6 +350,12 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( return -1; } + /* Fire the flushdb modules event. */ + RedisModuleFlushInfoV1 fi = {REDISMODULE_FLUSHINFO_VERSION,!async,dbnum}; + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_START, + &fi); + /* Make sure the WATCHed keys are affected by the FLUSH* commands. * Note that we need to call the function while the keys are still * there. */ @@ -380,6 +386,13 @@ long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)( } } if (dbnum == -1) flushSlaveKeysWithExpireList(); + + /* Also fire the end event. Note that this event will fire almost + * immediately after the start event if the flush is asynchronous. */ + moduleFireServerEvent(REDISMODULE_EVENT_FLUSHDB, + REDISMODULE_SUBEVENT_FLUSHDB_END, + &fi); + return removed; } diff --git a/src/module.c b/src/module.c index 8957849fd..f3fe4e9ac 100644 --- a/src/module.c +++ b/src/module.c @@ -5721,8 +5721,8 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * because of replication, after the replica synchronization) * happened. The following sub events are available: * - * REDISMODULE_EVENT_FLUSHALL_START - * REDISMODULE_EVENT_FLUSHALL_END + * REDISMODULE_EVENT_FLUSHDB_START + * REDISMODULE_EVENT_FLUSHDB_END * * The data pointer can be casted to a RedisModuleFlushInfo * structure with the following fields: diff --git a/src/redismodule.h b/src/redismodule.h index d8f4fb901..377ea7e13 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -239,6 +239,9 @@ static RedisModuleEvent #define REDISMODULE_SUBEVENT_REPLICA_CHANGE_CONNECTED 0 #define REDISMODULE_SUBEVENT_REPLICA_CHANGE_DISCONNECTED 1 +#define REDISMODULE_SUBEVENT_FLUSHDB_START 0 +#define REDISMODULE_SUBEVENT_FLUSHDB_END 1 + /* RedisModuleClientInfo flags. */ #define REDISMODULE_CLIENTINFO_FLAG_SSL (1<<0) #define REDISMODULE_CLIENTINFO_FLAG_PUBSUB (1<<1) From 77a37cafd5eae21b661ac7333ab5832ca52b511a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:37:04 +0200 Subject: [PATCH 205/320] Modules hooks: FLUSHDB event example. --- src/module.c | 2 ++ src/modules/hellohook.c | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/module.c b/src/module.c index f3fe4e9ac..a505a2099 100644 --- a/src/module.c +++ b/src/module.c @@ -5870,6 +5870,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); moduledata = &civ1; + } else if (eid == REDISMODULE_EVENT_FLUSHDB) { + moduledata = data; } el->callback(&ctx,el->event,subid,moduledata); moduleFreeContext(&ctx); diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c index ea0ac517d..2e798f500 100644 --- a/src/modules/hellohook.c +++ b/src/modules/hellohook.c @@ -50,6 +50,31 @@ void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, ci->id,ci->addr,ci->port); } +void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleFlushInfo *fi = data; + if (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) { + if (fi->dbnum != -1) { + RedisModuleCallReply *reply; + reply = RedisModule_Call(ctx,"DBSIZE",""); + long long numkeys = RedisModule_CallReplyInteger(reply); + printf("FLUSHDB event of database %d started (%lld keys in DB)\n", + fi->dbnum, numkeys); + } else { + printf("FLUSHALL event started\n"); + } + } else { + if (fi->dbnum != -1) { + printf("FLUSHDB event of database %d ended\n",fi->dbnum); + } else { + printf("FLUSHALL event ended\n"); + } + } +} + /* This function must be present on each Redis module. It is used in order to * register the commands into the Redis server. */ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { @@ -61,5 +86,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) RedisModule_SubscribeToServerEvent(ctx, RedisModuleEvent_ClientChange, clientChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_FlushDB, flushdbCallback); return REDISMODULE_OK; } From 9d10d62003b74fdc414b23fd6a708af3f410d0c9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:41:55 +0200 Subject: [PATCH 206/320] Modules hooks: select the right DB for FLUSHDB events. --- src/module.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/module.c b/src/module.c index a505a2099..70a63fee7 100644 --- a/src/module.c +++ b/src/module.c @@ -5872,6 +5872,9 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { moduledata = &civ1; } else if (eid == REDISMODULE_EVENT_FLUSHDB) { moduledata = data; + RedisModuleFlushInfoV1 *fi = data; + if (fi->dbnum != -1) + selectDb(ctx.client, fi->dbnum); } el->callback(&ctx,el->event,subid,moduledata); moduleFreeContext(&ctx); From d6739df4fcdd0de4c895cf0558ce10ca0bffb12d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:47:39 +0200 Subject: [PATCH 207/320] Modules hooks: do not re-enter in hooks. Calling a module hook callback may result in callback operations in turn triggering other events the module is subscribed too. We don't want to trigger those, it's unsafe and quite confusing, and to do it correcly we would need to maintain an event list: quite a more complex implementation. --- src/module.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 70a63fee7..27a4e96c1 100644 --- a/src/module.c +++ b/src/module.c @@ -61,6 +61,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ + int in_hook; /* Non zero if an hook callback is active. */ int options; /* Module options and capabilities. */ RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; @@ -817,6 +818,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->using = listCreate(); module->filters = listCreate(); module->in_call = 0; + module->in_hook = 0; ctx->module = module; } @@ -5859,7 +5861,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { listRewind(RedisModule_EventListeners,&li); while((ln = listNext(&li))) { RedisModuleEventListener *el = ln->value; - if (el->event.id == eid) { + if (el->event.id == eid && !el->module->in_hook) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.module = el->module; ctx.client = moduleFreeContextReusedClient; @@ -5876,7 +5878,9 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { if (fi->dbnum != -1) selectDb(ctx.client, fi->dbnum); } + el->module->in_hook = 1; el->callback(&ctx,el->event,subid,moduledata); + el->module->in_hook = 0; moduleFreeContext(&ctx); } } From 261a66b7915654a12eea62b8362610a8844e0733 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 10:52:25 +0200 Subject: [PATCH 208/320] Modules hooks: don't call the hook for fake clients. --- src/networking.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index dec49642f..e7cc561fa 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1043,9 +1043,13 @@ void freeClient(client *c) { freeClientAsync(c); return; } - moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, - REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, - c); + + /* For connected clients, call the disconnection event of modules hooks. */ + if (c->conn) { + moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE, + REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED, + c); + } /* If it is our master that's beging disconnected we should make sure * to cache the state to try a partial resynchronization later. From 938dafa9e32f395154a81aa37655f58620d5e0f5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 11:05:33 +0200 Subject: [PATCH 209/320] Modules hooks: for nested calls, create new fake clients. We can't use the same client at the same time when re-entering the hook. --- src/module.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/module.c b/src/module.c index 27a4e96c1..791e82e48 100644 --- a/src/module.c +++ b/src/module.c @@ -61,7 +61,7 @@ struct RedisModule { list *using; /* List of modules we use some APIs of. */ list *filters; /* List of filters the module has registered. */ int in_call; /* RM_Call() nesting level */ - int in_hook; /* Non zero if an hook callback is active. */ + int in_hook; /* Hooks callback nesting level for this module (0 or 1). */ int options; /* Module options and capabilities. */ RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; @@ -333,6 +333,8 @@ typedef struct RedisModuleEventListener { } RedisModuleEventListener; list *RedisModule_EventListeners; /* Global list of all the active events. */ +unsigned long long ModulesInHooks = 0; /* Total number of modules in hooks + callbacks right now. */ /* -------------------------------------------------------------------------- * Prototypes @@ -5864,7 +5866,14 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { if (el->event.id == eid && !el->module->in_hook) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.module = el->module; - ctx.client = moduleFreeContextReusedClient; + + if (ModulesInHooks == 0) { + ctx.client = moduleFreeContextReusedClient; + } else { + ctx.client = createClient(NULL); + ctx.client->flags |= CLIENT_MODULE; + ctx.client->user = NULL; /* Root user. */ + } void *moduledata = NULL; RedisModuleClientInfoV1 civ1; @@ -5878,9 +5887,14 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { if (fi->dbnum != -1) selectDb(ctx.client, fi->dbnum); } - el->module->in_hook = 1; + + ModulesInHooks++; + el->module->in_hook++; el->callback(&ctx,el->event,subid,moduledata); - el->module->in_hook = 0; + el->module->in_hook--; + ModulesInHooks--; + + if (ModulesInHooks != 0) freeClient(ctx.client); moduleFreeContext(&ctx); } } From 83adff66c72020626116e328af068d2ecedf342d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 18:21:57 +0200 Subject: [PATCH 210/320] Modules hooks: fix memory leak in example module. --- src/modules/hellohook.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modules/hellohook.c b/src/modules/hellohook.c index 2e798f500..7ab78ed07 100644 --- a/src/modules/hellohook.c +++ b/src/modules/hellohook.c @@ -63,6 +63,7 @@ void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void long long numkeys = RedisModule_CallReplyInteger(reply); printf("FLUSHDB event of database %d started (%lld keys in DB)\n", fi->dbnum, numkeys); + RedisModule_FreeCallReply(reply); } else { printf("FLUSHALL event started\n"); } From e9f6688a68f978993e7489e5c62ae561076aec21 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 18:38:30 +0200 Subject: [PATCH 211/320] Modules hooks: test module draft. --- tests/modules/Makefile | 6 +++- tests/modules/hooks.c | 77 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 tests/modules/hooks.c diff --git a/tests/modules/Makefile b/tests/modules/Makefile index c1c8ffa2e..988ecf58a 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: commandfilter.so testrdb.so fork.so infotest.so propagate.so +all: commandfilter.so testrdb.so fork.so infotest.so propagate.so hooks.so .c.xo: $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -23,6 +23,7 @@ fork.xo: ../../src/redismodule.h testrdb.xo: ../../src/redismodule.h infotest.xo: ../../src/redismodule.h propagate.xo: ../../src/redismodule.h +hooks.xo: ../../src/redismodule.h commandfilter.so: commandfilter.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc @@ -38,3 +39,6 @@ infotest.so: infotest.xo propagate.so: propagate.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc + +hooks.so: hooks.xo + $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc diff --git a/tests/modules/hooks.c b/tests/modules/hooks.c new file mode 100644 index 000000000..f23f8a2a4 --- /dev/null +++ b/tests/modules/hooks.c @@ -0,0 +1,77 @@ +/* This module is used to test the server events hooks API. + * + * ----------------------------------------------------------------------------- + * + * Copyright (c) 2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +/* Client state change callback. */ +void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleClientInfo *ci = data; + char *keyname = (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? + "connected" : "disconnected"; + RedisModuleCallReply *reply; + reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)ci->id); + RedisModule_FreeCallReply(reply); +} + +void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(e); + + RedisModuleFlushInfo *fi = data; + char *keyname = (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) ? + "flush-start" : "flush-end"; + RedisModuleCallReply *reply; + reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)fi->dbnum); + RedisModule_FreeCallReply(reply); +} + +/* This function must be present on each Redis module. It is used in order to + * register the commands into the Redis server. */ +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"testhook",1,REDISMODULE_APIVER_1) + == REDISMODULE_ERR) return REDISMODULE_ERR; + + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ClientChange, clientChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_FlushDB, flushdbCallback); + return REDISMODULE_OK; +} From a39806662552f8f643b942ad353db44130647710 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Oct 2019 18:51:44 +0200 Subject: [PATCH 212/320] Modules hooks: select DB zero by default. --- src/module.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/module.c b/src/module.c index 791e82e48..012ef8001 100644 --- a/src/module.c +++ b/src/module.c @@ -5877,6 +5877,11 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { void *moduledata = NULL; RedisModuleClientInfoV1 civ1; + /* Start at DB zero by default when calling the handler. It's + * up to the specific event setup to change it when it makes + * sense. For instance for FLUSHDB events we select the correct + * DB automatically. */ + selectDb(ctx.client, 0); if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); From 70815145a7284b31390e8a34676145cb71275296 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 24 Oct 2019 09:45:25 +0300 Subject: [PATCH 213/320] Make module tests pass with valgrind, and fix a leak in diskless load --- src/rdb.c | 6 ++++- tests/modules/commandfilter.c | 5 ++++ tests/modules/testrdb.c | 8 ++++++ tests/unit/moduleapi/testrdb.tcl | 45 +++++++++++++------------------- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 2406ea88a..cbaa428a1 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2172,7 +2172,11 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { /* Read key */ if ((key = rdbLoadStringObject(rdb)) == NULL) goto eoferr; /* Read value */ - if ((val = rdbLoadObject(type,rdb,key)) == NULL) goto eoferr; + if ((val = rdbLoadObject(type,rdb,key)) == NULL) { + decrRefCount(key); + goto eoferr; + } + /* Check if the key already expired. This function is used when loading * an RDB file from disk, either at startup, or when an RDB was * received from the master. In the latter case, the master is diff --git a/tests/modules/commandfilter.c b/tests/modules/commandfilter.c index d25d49c44..571ed1701 100644 --- a/tests/modules/commandfilter.c +++ b/tests/modules/commandfilter.c @@ -147,3 +147,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } + +int RedisModule_OnUnload(RedisModuleCtx *ctx) { + RedisModule_FreeString(ctx, log_key_name); + return REDISMODULE_OK; +} diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index d73c8bfd3..eb8d1a999 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -238,3 +238,11 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } + +int RedisModule_OnUnload(RedisModuleCtx *ctx) { + if (before_str) + RedisModule_FreeString(ctx, before_str); + if (after_str) + RedisModule_FreeString(ctx, after_str); + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/testrdb.tcl b/tests/unit/moduleapi/testrdb.tcl index c72570002..a93b34b69 100644 --- a/tests/unit/moduleapi/testrdb.tcl +++ b/tests/unit/moduleapi/testrdb.tcl @@ -1,57 +1,48 @@ set testmodule [file normalize tests/modules/testrdb.so] -proc restart_and_wait {} { - catch { - r debug restart - } - - # wait for the server to come back up - set retry 50 - while {$retry} { - if {[catch { r ping }]} { - after 100 - } else { - break - } - incr retry -1 - } -} - tags "modules" { - start_server [list overrides [list loadmodule "$testmodule"]] { - test {modules are able to persist types} { + test {modules are able to persist types} { + start_server [list overrides [list loadmodule "$testmodule"]] { r testrdb.set.key key1 value1 assert_equal "value1" [r testrdb.get.key key1] r debug reload assert_equal "value1" [r testrdb.get.key key1] } + } - test {modules global are lost without aux} { + test {modules global are lost without aux} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path]] { r testrdb.set.before global1 assert_equal "global1" [r testrdb.get.before] - restart_and_wait + } + start_server [list overrides [list loadmodule "$testmodule" "dir" $server_path]] { assert_equal "" [r testrdb.get.before] } } - start_server [list overrides [list loadmodule "$testmodule 2"]] { - test {modules are able to persist globals before and after} { + test {modules are able to persist globals before and after} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule 2" "dir" $server_path]] { r testrdb.set.before global1 r testrdb.set.after global2 assert_equal "global1" [r testrdb.get.before] assert_equal "global2" [r testrdb.get.after] - restart_and_wait + } + start_server [list overrides [list loadmodule "$testmodule 2" "dir" $server_path]] { assert_equal "global1" [r testrdb.get.before] assert_equal "global2" [r testrdb.get.after] } } - start_server [list overrides [list loadmodule "$testmodule 1"]] { - test {modules are able to persist globals just after} { + test {modules are able to persist globals just after} { + set server_path [tmpdir "server.module-testrdb"] + start_server [list overrides [list loadmodule "$testmodule 1" "dir" $server_path]] { r testrdb.set.after global2 assert_equal "global2" [r testrdb.get.after] - restart_and_wait + } + start_server [list overrides [list loadmodule "$testmodule 1" "dir" $server_path]] { assert_equal "global2" [r testrdb.get.after] } } From 51b680be424fc244c2703e0a1ad36e9fc4e72500 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Oct 2019 10:41:25 +0200 Subject: [PATCH 214/320] Modules hooks: initial Tcl test file. --- runtest-moduleapi | 2 +- tests/unit/moduleapi/hooks.tcl | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 tests/unit/moduleapi/hooks.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index a16cca686..444204919 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,4 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate "${@}" +$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate --single unit/moduleapi/hooks "${@}" diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl new file mode 100644 index 000000000..473c8fdd1 --- /dev/null +++ b/tests/unit/moduleapi/hooks.tcl @@ -0,0 +1,18 @@ +set testmodule [file normalize tests/modules/hooks.so] + +tags "modules" { + start_server {} { + r module load $testmodule + test {Test clients connection / disconnection hooks} { + for {set j 0} {$j < 2} {incr j} { + set rd1 [redis_deferring_client] + $rd1 close + } + + r select 0 + puts "Keys: [r keys *]" + assert {[r llen connected] > 1} + assert {[r llen disconnected] > 1} + } + } +} From 3bd7f715c8a185fc3697fa9623de58a78a529cc0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Oct 2019 10:51:03 +0200 Subject: [PATCH 215/320] Modules hooks: test flush event. --- src/module.c | 2 ++ tests/modules/hooks.c | 2 ++ tests/unit/moduleapi/hooks.tcl | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 012ef8001..cabb37e08 100644 --- a/src/module.c +++ b/src/module.c @@ -5882,6 +5882,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { * sense. For instance for FLUSHDB events we select the correct * DB automatically. */ selectDb(ctx.client, 0); + + /* Event specific context and data pointer setup. */ if (eid == REDISMODULE_EVENT_CLIENT_CHANGE) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); diff --git a/tests/modules/hooks.c b/tests/modules/hooks.c index f23f8a2a4..33b690b2f 100644 --- a/tests/modules/hooks.c +++ b/tests/modules/hooks.c @@ -43,6 +43,7 @@ void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, char *keyname = (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? "connected" : "disconnected"; RedisModuleCallReply *reply; + RedisModule_SelectDb(ctx,9); reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)ci->id); RedisModule_FreeCallReply(reply); } @@ -56,6 +57,7 @@ void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void char *keyname = (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) ? "flush-start" : "flush-end"; RedisModuleCallReply *reply; + RedisModule_SelectDb(ctx,9); reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)fi->dbnum); RedisModule_FreeCallReply(reply); } diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl index 473c8fdd1..7a727902d 100644 --- a/tests/unit/moduleapi/hooks.tcl +++ b/tests/unit/moduleapi/hooks.tcl @@ -8,11 +8,21 @@ tags "modules" { set rd1 [redis_deferring_client] $rd1 close } - - r select 0 - puts "Keys: [r keys *]" assert {[r llen connected] > 1} assert {[r llen disconnected] > 1} } + + test {Test flushdb hooks} { + r flushall ;# Note: only the "end" RPUSH will survive + r select 1 + r flushdb + r select 2 + r flushdb + r select 9 + assert {[r llen flush-start] == 2} + assert {[r llen flush-end] == 3} + assert {[r lrange flush-start 0 -1] eq {1 2}} + assert {[r lrange flush-end 0 -1] eq {-1 1 2}} + } } } From 1ffe4578e2f0f9ede7e41c5a3df1db6855949212 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Oct 2019 11:20:15 +0200 Subject: [PATCH 216/320] Remove trailing space from server.c. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index d16ff0a8e..cbcbf7183 100644 --- a/src/server.c +++ b/src/server.c @@ -146,7 +146,7 @@ volatile unsigned long lru_clock; /* Server global current LRU time. */ * in this condition but just a few. * * no-monitor: Do not automatically propagate the command on MONITOR. - * + * * no-slowlog: Do not automatically propagate the command to the slowlog. * * cluster-asking: Perform an implicit ASKING for this command, so the From a6d42ebfd93c8869196a51175702a3e9794abdf7 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 23 Oct 2019 19:32:35 +0300 Subject: [PATCH 217/320] Modules: improve tests Makefile. Removes some boilerplate per module, add clean target. --- tests/modules/Makefile | 39 +++++++++++++++------------------------ 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 988ecf58a..f357faad2 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -11,34 +11,25 @@ else SHOBJ_LDFLAGS ?= -bundle -undefined dynamic_lookup endif -.SUFFIXES: .c .so .xo .o +TEST_MODULES = \ + commandfilter.so \ + testrdb.so \ + fork.so \ + infotest.so \ + propagate.so \ + hooks.so -all: commandfilter.so testrdb.so fork.so infotest.so propagate.so hooks.so +.PHONY: all -.c.xo: +all: $(TEST_MODULES) + +%.xo: %.c ../../src/redismodule.h $(CC) -I../../src $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ -commandfilter.xo: ../../src/redismodule.h -fork.xo: ../../src/redismodule.h -testrdb.xo: ../../src/redismodule.h -infotest.xo: ../../src/redismodule.h -propagate.xo: ../../src/redismodule.h -hooks.xo: ../../src/redismodule.h - -commandfilter.so: commandfilter.xo +%.so: %.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc -fork.so: fork.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc +.PHONY: clean -testrdb.so: testrdb.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc - -infotest.so: infotest.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc - -propagate.so: propagate.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc - -hooks.so: hooks.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc +clean: + rm -f $(TEST_MODULES) $(TEST_MODULES:.so=.xo) From 5ba6fe50c70095e2d25077a9c77b765aba785381 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 24 Oct 2019 14:24:55 +0300 Subject: [PATCH 218/320] Module API for LatencyAddSample --- src/latency.c | 2 +- src/latency.h | 2 +- src/module.c | 9 +++++++++ src/redismodule.h | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/latency.c b/src/latency.c index b834da5c7..74ced72a5 100644 --- a/src/latency.c +++ b/src/latency.c @@ -95,7 +95,7 @@ void latencyMonitorInit(void) { * This function is usually called via latencyAddSampleIfNeeded(), that * is a macro that only adds the sample if the latency is higher than * server.latency_monitor_threshold. */ -void latencyAddSample(char *event, mstime_t latency) { +void latencyAddSample(const char *event, mstime_t latency) { struct latencyTimeSeries *ts = dictFetchValue(server.latency_events,event); time_t now = time(NULL); int prev; diff --git a/src/latency.h b/src/latency.h index 0fe26e0e4..76640cfce 100644 --- a/src/latency.h +++ b/src/latency.h @@ -62,7 +62,7 @@ struct latencyStats { }; void latencyMonitorInit(void); -void latencyAddSample(char *event, mstime_t latency); +void latencyAddSample(const char *event, mstime_t latency); int THPIsEnabled(void); /* Latency monitoring macros. */ diff --git a/src/module.c b/src/module.c index ff5eba787..1fccaabce 100644 --- a/src/module.c +++ b/src/module.c @@ -3793,6 +3793,14 @@ void RM__Assert(const char *estr, const char *file, int line) { _serverAssert(estr, file, line); } +/* Allows adding event to the latency monitor to be observed by the LATENCY + * command. The call is skipped if the latency is smaller than the configured + * latency-monitor-threshold. */ +void RM_LatencyAddSample(const char *event, mstime_t latency) { + if (latency >= server.latency_monitor_threshold) + latencyAddSample(event, latency); +} + /* -------------------------------------------------------------------------- * Blocking clients from modules * -------------------------------------------------------------------------- */ @@ -6024,6 +6032,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(Log); REGISTER_API(LogIOError); REGISTER_API(_Assert); + REGISTER_API(LatencyAddSample); REGISTER_API(StringAppendBuffer); REGISTER_API(RetainString); REGISTER_API(StringCompare); diff --git a/src/redismodule.h b/src/redismodule.h index 19a9cd897..5dc3cb61b 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -319,6 +319,7 @@ float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); +void REDISMODULE_API_FUNC(RedisModule_LatencyAddSample)(const char *event, mstime_t latency); int REDISMODULE_API_FUNC(RedisModule_StringAppendBuffer)(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len); void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisModuleString *str); int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); @@ -509,6 +510,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); REDISMODULE_GET_API(_Assert); + REDISMODULE_GET_API(LatencyAddSample); REDISMODULE_GET_API(StringAppendBuffer); REDISMODULE_GET_API(RetainString); REDISMODULE_GET_API(StringCompare); From fd5c1adbd7608a8994c87b45b9c65693e5402cc5 Mon Sep 17 00:00:00 2001 From: happynote3966 <01ssrmikururudevice01@gmail.com> Date: Fri, 25 Oct 2019 22:31:55 +0900 Subject: [PATCH 219/320] fix comment typo in redis-cli.c --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 6d07f7ba6..c67663c23 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -370,7 +370,7 @@ static sds percentDecode(const char *pe, size_t len) { * URI scheme is based on the the provisional specification[1] excluding support * for query parameters. Valid URIs are: * scheme: "redis://" - * authority: [ ":"] "@"] [ [":" ]] + * authority: [[ ":"] "@"] [ [":" ]] * path: ["/" []] * * [1]: https://www.iana.org/assignments/uri-schemes/prov/redis */ From f12c08d921947c2765bfcc566287df5acbcc9e05 Mon Sep 17 00:00:00 2001 From: Carsten Strotmann Date: Fri, 25 Oct 2019 19:38:45 +0200 Subject: [PATCH 220/320] Typo fix: kill -> still --- redis.conf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/redis.conf b/redis.conf index 408426f15..0ec3321a5 100644 --- a/redis.conf +++ b/redis.conf @@ -134,7 +134,7 @@ tcp-keepalive 300 # By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration # directive can be used to define TLS-listening ports. To enable TLS on the # default port, use: -# +# # port 0 # tls-port 6379 @@ -1309,7 +1309,7 @@ notify-keyspace-events "" # Redis contains an implementation of the Gopher protocol, as specified in # the RFC 1436 (https://www.ietf.org/rfc/rfc1436.txt). # -# The Gopher protocol was very popular in the late '90s. It is an alternative +# The Gopher protocol was very popular in the late '90s. It is an alternative # to the web, and the implementation both server and client side is so simple # that the Redis server has just 100 lines of code in order to implement this # support. @@ -1347,7 +1347,7 @@ notify-keyspace-events "" # to server Gopher pages MAKE SURE TO SET A PASSWORD to the instance. # Once a password is set: # -# 1. The Gopher server (when enabled, not by default) will kill serve +# 1. The Gopher server (when enabled, not by default) will still serve # content via Gopher. # 2. However other commands cannot be called before the client will # authenticate. @@ -1669,4 +1669,3 @@ rdb-save-incremental-fsync yes # Maximum number of set/hash/zset/list fields that will be processed from # the main dictionary scan # active-defrag-max-scan-fields 1000 - From 18c4ccf8986264d9e86764399622bd2949bdf9e5 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 8 Jun 2018 10:33:37 +0800 Subject: [PATCH 221/320] Modules: make unloading module more safe As we know if a module exports module-side data types, unload it is not allowed. This rule is the same with blocked clients in module, because we use background threads to implement module blocked clients, and it's not safe to unload a module if there are background threads running. So it's necessary to check if any blocked clients running in this module when unload it. Moreover, after that we can ensure that if no modules, then no module blocked clients even module unloaded. So, we can call moduleHandleBlockedClients only when we have installed modules. --- src/module.c | 11 +++++++++++ src/server.c | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index cabb37e08..eb994a8e5 100644 --- a/src/module.c +++ b/src/module.c @@ -63,6 +63,7 @@ struct RedisModule { int in_call; /* RM_Call() nesting level */ int in_hook; /* Hooks callback nesting level for this module (0 or 1). */ int options; /* Module options and capabilities. */ + int blocked_clients; /* Count of RedisModuleBlockedClient in this module. */ RedisModuleInfoFunc info_cb; /* Callback for module to add INFO fields. */ }; typedef struct RedisModule RedisModule; @@ -3961,6 +3962,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + ctx->module->blocked_clients++; /* We need to handle the invalid operation of calling modules blocking * commands from Lua or MULTI. We actually create an already aborted @@ -4119,6 +4121,7 @@ void moduleHandleBlockedClients(void) { /* Free 'bc' only after unblocking the client, since it is * referenced in the client blocking context, and must be valid * when calling unblockClient(). */ + bc->module->blocked_clients--; zfree(bc); /* Lock again before to iterate the loop. */ @@ -6089,6 +6092,7 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { /* Redis module loaded! Register it. */ dictAdd(modules,ctx.module->name,ctx.module); + ctx.module->blocked_clients = 0; ctx.module->handle = handle; serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path); moduleFreeContext(&ctx); @@ -6114,6 +6118,9 @@ int moduleUnload(sds name) { } else if (listLength(module->usedby)) { errno = EPERM; return REDISMODULE_ERR; + } else if (module->blocked_clients) { + errno = EAGAIN; + return REDISMODULE_ERR; } /* Give module a chance to clean up. */ @@ -6279,6 +6286,10 @@ NULL errmsg = "the module exports APIs used by other modules. " "Please unload them first and try again"; break; + case EAGAIN: + errmsg = "the module has blocked clients. " + "Please wait them unblocked and try again"; + break; default: errmsg = "operation not possible."; break; diff --git a/src/server.c b/src/server.c index d16ff0a8e..77db810b2 100644 --- a/src/server.c +++ b/src/server.c @@ -2104,7 +2104,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { /* Check if there are clients unblocked by modules that implement * blocking commands. */ - moduleHandleBlockedClients(); + if (moduleCount()) moduleHandleBlockedClients(); /* Try to process pending commands for clients that were just unblocked. */ if (listLength(server.unblocked_clients)) From 29827d37a52306140a6c550ddb1b9154c1ba02ea Mon Sep 17 00:00:00 2001 From: swilly22 Date: Mon, 28 Oct 2019 08:50:25 +0200 Subject: [PATCH 222/320] Introduce ReplyWithVerbatimString, ReplyWithEmptyArray, ReplyWithNullArray and ReplyWithEmptyString to redis module API --- src/module.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/redismodule.h | 9 ++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index cabb37e08..fe2a0de28 100644 --- a/src/module.c +++ b/src/module.c @@ -1264,6 +1264,27 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { return REDISMODULE_OK; } +/* Reply to the client with a null array, simply null in RESP3 + * null array in RESP2. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithNullArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyNullArray(c); + return REDISMODULE_OK; +} + +/* Reply to the client with an empty array. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyArray(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReply(c,shared.emptyarray); + return REDISMODULE_OK; +} + /* When RedisModule_ReplyWithArray() is used with the argument * REDISMODULE_POSTPONED_ARRAY_LEN, because we don't know beforehand the number * of items we are going to output as elements of the array, this function @@ -1342,6 +1363,27 @@ int RM_ReplyWithString(RedisModuleCtx *ctx, RedisModuleString *str) { return REDISMODULE_OK; } +/* Reply with an empty string. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithEmptyString(RedisModuleCtx *ctx) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyBulkCBuffer(c, "", 0); + return REDISMODULE_OK; +} + +/* Reply with a binary safe string, which should not be escaped or filtered + * taking in input a C buffer pointer and length. + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithVerbatimString(RedisModuleCtx *ctx, const char *buf, size_t len) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyVerbatim(c, buf, len, "txt"); + return REDISMODULE_OK; +} + /* Reply to the client with a NULL. In the RESP protocol a NULL is encoded * as the string "$-1\r\n". * @@ -6316,8 +6358,12 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplyWithError); REGISTER_API(ReplyWithSimpleString); REGISTER_API(ReplyWithArray); + REGISTER_API(ReplyWithNullArray); + REGISTER_API(ReplyWithEmptyArray); REGISTER_API(ReplySetArrayLength); REGISTER_API(ReplyWithString); + REGISTER_API(ReplyWithEmptyString); + REGISTER_API(ReplyWithVerbatimString); REGISTER_API(ReplyWithStringBuffer); REGISTER_API(ReplyWithCString); REGISTER_API(ReplyWithNull); diff --git a/src/redismodule.h b/src/redismodule.h index 377ea7e13..9308e395c 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -382,10 +382,14 @@ const char *REDISMODULE_API_FUNC(RedisModule_StringPtrLen)(const RedisModuleStri int REDISMODULE_API_FUNC(RedisModule_ReplyWithError)(RedisModuleCtx *ctx, const char *err); int REDISMODULE_API_FUNC(RedisModule_ReplyWithSimpleString)(RedisModuleCtx *ctx, const char *msg); int REDISMODULE_API_FUNC(RedisModule_ReplyWithArray)(RedisModuleCtx *ctx, long len); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithNullArray)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyArray)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_ReplySetArrayLength)(RedisModuleCtx *ctx, long len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithStringBuffer)(RedisModuleCtx *ctx, const char *buf, size_t len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithCString)(RedisModuleCtx *ctx, const char *buf); int REDISMODULE_API_FUNC(RedisModule_ReplyWithString)(RedisModuleCtx *ctx, RedisModuleString *str); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyString)(RedisModuleCtx *ctx); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithVerbatimString)(RedisModuleCtx *ctx, const char *buf, size_t len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d); int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply); @@ -552,14 +556,17 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ReplyWithError); REDISMODULE_GET_API(ReplyWithSimpleString); REDISMODULE_GET_API(ReplyWithArray); + REDISMODULE_GET_API(ReplyWithNullArray); + REDISMODULE_GET_API(ReplyWithEmptyArray); REDISMODULE_GET_API(ReplySetArrayLength); REDISMODULE_GET_API(ReplyWithStringBuffer); REDISMODULE_GET_API(ReplyWithCString); REDISMODULE_GET_API(ReplyWithString); + REDISMODULE_GET_API(ReplyWithEmptyString); + REDISMODULE_GET_API(ReplyWithVerbatimString); REDISMODULE_GET_API(ReplyWithNull); REDISMODULE_GET_API(ReplyWithCallReply); REDISMODULE_GET_API(ReplyWithDouble); - REDISMODULE_GET_API(ReplySetArrayLength); REDISMODULE_GET_API(GetSelectedDb); REDISMODULE_GET_API(SelectDb); REDISMODULE_GET_API(OpenKey); From 6700a2dc668ebefe058d26c4f272a89c1e0c4db6 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 23 Oct 2019 12:08:47 +0300 Subject: [PATCH 223/320] Module API for explicit SignalModifiedKey instead of implicit one. This commit also fixes an uninitialized module struct member (that luckily never got released) --- src/module.c | 19 ++++++++++++++++++- src/redismodule.h | 6 ++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 971bf5c08..21bab5cdf 100644 --- a/src/module.c +++ b/src/module.c @@ -322,6 +322,13 @@ static struct RedisModuleForkInfo { #define REDISMODULE_ARGV_NO_AOF (1<<1) #define REDISMODULE_ARGV_NO_REPLICAS (1<<2) +/* Determine whether Redis should signalModifiedKey implicitly. + * In case 'ctx' has no 'module' member (and therefore no module->options), + * we assume default behavior, that is, Redis signals. + * (see RM_GetThreadSafeContext) */ +#define SHOULD_SIGNAL_MODIFIED_KEYS(ctx) \ + ctx->module? !(ctx->module->options & REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED) : 1 + /* Server events hooks data structures and defines: this modules API * allow modules to subscribe to certain events in Redis, such as * the start and end of an RDB or AOF save, the change of role in replication, @@ -822,6 +829,7 @@ void RM_SetModuleAttribs(RedisModuleCtx *ctx, const char *name, int ver, int api module->filters = listCreate(); module->in_call = 0; module->in_hook = 0; + module->options = 0; ctx->module = module; } @@ -852,6 +860,12 @@ void RM_SetModuleOptions(RedisModuleCtx *ctx, int options) { ctx->module->options = options; } +/* Signals that the key is modified from user's perspective (i.e. invalidate WATCH). */ +int RM_SignalModifiedKey(RedisModuleCtx *ctx, RedisModuleString *keyname) { + signalModifiedKey(ctx->client->db,keyname); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Automatic memory management for modules * -------------------------------------------------------------------------- */ @@ -1843,7 +1857,9 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { /* Close a key handle. */ void RM_CloseKey(RedisModuleKey *key) { if (key == NULL) return; - if (key->mode & REDISMODULE_WRITE) signalModifiedKey(key->db,key->key); + int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); + if ((key->mode & REDISMODULE_WRITE) && signal) + signalModifiedKey(key->db,key->key); /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ RM_ZsetRangeStop(key); decrRefCount(key->key); @@ -6449,6 +6465,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ModuleTypeGetValue); REGISTER_API(IsIOError); REGISTER_API(SetModuleOptions); + REGISTER_API(SignalModifiedKey); REGISTER_API(SaveUnsigned); REGISTER_API(LoadUnsigned); REGISTER_API(SaveSigned); diff --git a/src/redismodule.h b/src/redismodule.h index 4b63a227c..9fe0f5eba 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -161,6 +161,10 @@ typedef uint64_t RedisModuleTimerID; /* Declare that the module can handle errors with RedisModule_SetModuleOptions. */ #define REDISMODULE_OPTIONS_HANDLE_IO_ERRORS (1<<0) +/* When set, Redis will not call RedisModule_SignalModifiedKey(), implicitly in + * RedisModule_CloseKey, and the module needs to do that when manually when keys + * are modified from the user's sperspective, to invalidate WATCH. */ +#define REDISMODULE_OPTION_NO_IMPLICIT_SIGNAL_MODIFIED (1<<1) /* Server events definitions. */ #define REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED 0 @@ -434,6 +438,7 @@ RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModule void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SetModuleOptions)(RedisModuleCtx *ctx, int options); +int REDISMODULE_API_FUNC(RedisModule_SignalModifiedKey)(RedisModuleCtx *ctx, RedisModuleString *keyname); void REDISMODULE_API_FUNC(RedisModule_SaveUnsigned)(RedisModuleIO *io, uint64_t value); uint64_t REDISMODULE_API_FUNC(RedisModule_LoadUnsigned)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveSigned)(RedisModuleIO *io, int64_t value); @@ -629,6 +634,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ModuleTypeGetValue); REDISMODULE_GET_API(IsIOError); REDISMODULE_GET_API(SetModuleOptions); + REDISMODULE_GET_API(SignalModifiedKey); REDISMODULE_GET_API(SaveUnsigned); REDISMODULE_GET_API(LoadUnsigned); REDISMODULE_GET_API(SaveSigned); From ac5f73bfbf50c4ac7869ce7b2d38fedb1ca7c8b3 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2019 11:10:58 +0100 Subject: [PATCH 224/320] LOLWUT 6: always pick a default color sequence. --- src/lolwut6.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lolwut6.c b/src/lolwut6.c index b76d80690..471bf66c8 100644 --- a/src/lolwut6.c +++ b/src/lolwut6.c @@ -59,6 +59,7 @@ static sds renderCanvas(lwCanvas *canvas) { case 1: ce = "0;90;100m"; break; /* Gray 1 */ case 2: ce = "0;37;47m"; break; /* Gray 2 */ case 3: ce = "0;97;107m"; break; /* White */ + default: ce = "0;30;40m"; break; /* Just for safety. */ } text = sdscatprintf(text,"\033[%s \033[0m",ce); } From 5d01f01beb3857ad312bf9b706dd2b89f54a9d3c Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 15 Oct 2019 12:51:30 +0200 Subject: [PATCH 225/320] Modules: Allow notifying custom keyspace events Also, add an API for getting server.notify_keyspace_events Other (unrelated) changes: Add RM_GetKeynameFromModuleKey --- src/module.c | 22 ++++++++++++++++++++++ src/redismodule.h | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/src/module.c b/src/module.c index 971bf5c08..55568aca8 100644 --- a/src/module.c +++ b/src/module.c @@ -3859,6 +3859,11 @@ const RedisModuleString *RM_GetKeyNameFromIO(RedisModuleIO *io) { return io->key; } +/* Returns a RedisModuleString with the name of the key from RedisModuleKey */ +const RedisModuleString *RM_GetKeyNameFromModuleKey(RedisModuleKey *key) { + return key ? key->key : NULL; +} + /* -------------------------------------------------------------------------- * Logging * -------------------------------------------------------------------------- */ @@ -4368,6 +4373,20 @@ int RM_SubscribeToKeyspaceEvents(RedisModuleCtx *ctx, int types, RedisModuleNoti return REDISMODULE_OK; } +/* Get the configured bitmap of notify-keyspace-events (Could be used + * for additional filtering in RedisModuleNotificationFunc) */ +int RM_GetNotifyKeyspaceEvents() { + return server.notify_keyspace_events; +} + +/* Expose notifyKeyspaceEvent to modules */ +int RM_NotifyKeyspaceEvent(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) { + if (!ctx || !ctx->client) + return REDISMODULE_ERR; + notifyKeyspaceEvent(type, (char *)event, key, ctx->client->db->id); + return REDISMODULE_OK; +} + /* Dispatcher for keyspace notifications to module subscriber functions. * This gets called only if at least one module requested to be notified on * keyspace notifications */ @@ -6471,6 +6490,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StringCompare); REGISTER_API(GetContextFromIO); REGISTER_API(GetKeyNameFromIO); + REGISTER_API(GetKeyNameFromModuleKey); REGISTER_API(BlockClient); REGISTER_API(UnblockClient); REGISTER_API(IsBlockedReplyRequest); @@ -6485,6 +6505,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(DigestAddStringBuffer); REGISTER_API(DigestAddLongLong); REGISTER_API(DigestEndSequence); + REGISTER_API(NotifyKeyspaceEvent); + REGISTER_API(GetNotifyKeyspaceEvents); REGISTER_API(SubscribeToKeyspaceEvents); REGISTER_API(RegisterClusterMessageReceiver); REGISTER_API(SendClusterMessage); diff --git a/src/redismodule.h b/src/redismodule.h index 4b63a227c..434849c77 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -106,6 +106,11 @@ /* There is currently some background process active. */ #define REDISMODULE_CTX_FLAGS_ACTIVE_CHILD (1<<18) +/* Keyspace changes notification classes. Every class is associated with a + * character for configuration purposes. + * NOTE: These have to be in sync with NOTIFY_* in server.h */ +#define REDISMODULE_NOTIFY_KEYSPACE (1<<0) /* K */ +#define REDISMODULE_NOTIFY_KEYEVENT (1<<1) /* E */ #define REDISMODULE_NOTIFY_GENERIC (1<<2) /* g */ #define REDISMODULE_NOTIFY_STRING (1<<3) /* $ */ #define REDISMODULE_NOTIFY_LIST (1<<4) /* l */ @@ -651,6 +656,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StringCompare); REDISMODULE_GET_API(GetContextFromIO); REDISMODULE_GET_API(GetKeyNameFromIO); + REDISMODULE_GET_API(GetKeyNameFromModuleKey); REDISMODULE_GET_API(Milliseconds); REDISMODULE_GET_API(DigestAddStringBuffer); REDISMODULE_GET_API(DigestAddLongLong); @@ -703,6 +709,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(AbortBlock); REDISMODULE_GET_API(SetDisconnectCallback); REDISMODULE_GET_API(SubscribeToKeyspaceEvents); + REDISMODULE_GET_API(NotifyKeyspaceEvent); + REDISMODULE_GET_API(GetNotifyKeyspaceEvents); REDISMODULE_GET_API(BlockedClientDisconnected); REDISMODULE_GET_API(RegisterClusterMessageReceiver); REDISMODULE_GET_API(SendClusterMessage); From d7257888735480277cbbb6d470935b745a32800c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 24 Oct 2019 09:38:52 +0300 Subject: [PATCH 226/320] Module api tests for RM_Call Adding a test for coverage for RM_Call in a new "misc" unit to be used for various short simple tests also solves compilation warnings in redismodule.h and fork.c --- runtest-moduleapi | 10 ++++++- src/module.c | 5 +++- src/redismodule.h | 2 +- tests/modules/Makefile | 1 + tests/modules/fork.c | 6 +++- tests/modules/misc.c | 55 +++++++++++++++++++++++++++++++++++ tests/unit/moduleapi/misc.tcl | 19 ++++++++++++ 7 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 tests/modules/misc.c create mode 100644 tests/unit/moduleapi/misc.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 444204919..9301002c9 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -13,4 +13,12 @@ then fi make -C tests/modules && \ -$TCLSH tests/test_helper.tcl --single unit/moduleapi/commandfilter --single unit/moduleapi/fork --single unit/moduleapi/testrdb --single unit/moduleapi/infotest --single unit/moduleapi/propagate --single unit/moduleapi/hooks "${@}" +$TCLSH tests/test_helper.tcl \ +--single unit/moduleapi/commandfilter \ +--single unit/moduleapi/fork \ +--single unit/moduleapi/testrdb \ +--single unit/moduleapi/infotest \ +--single unit/moduleapi/propagate \ +--single unit/moduleapi/hooks \ +--single unit/moduleapi/misc \ +"${@}" diff --git a/src/module.c b/src/module.c index 971bf5c08..8236728b6 100644 --- a/src/module.c +++ b/src/module.c @@ -3067,7 +3067,10 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* We handle the above format error only when the client is setup so that * we can free it normally. */ - if (argv == NULL) goto cleanup; + if (argv == NULL) { + errno = EINVAL; + goto cleanup; + } /* Call command filters */ moduleCallCommandFilters(c); diff --git a/src/redismodule.h b/src/redismodule.h index 4b63a227c..1171f81b4 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -181,7 +181,7 @@ typedef struct RedisModuleEvent { struct RedisModuleCtx; typedef void (*RedisModuleEventCallback)(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64_t subevent, void *data); -static RedisModuleEvent +static const RedisModuleEvent RedisModuleEvent_ReplicationRoleChanged = { REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, 1 diff --git a/tests/modules/Makefile b/tests/modules/Makefile index f357faad2..71c0b5ef8 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -17,6 +17,7 @@ TEST_MODULES = \ fork.so \ infotest.so \ propagate.so \ + misc.so \ hooks.so .PHONY: all diff --git a/tests/modules/fork.c b/tests/modules/fork.c index 0804e4355..1a139ef1b 100644 --- a/tests/modules/fork.c +++ b/tests/modules/fork.c @@ -1,6 +1,10 @@ #define REDISMODULE_EXPERIMENTAL_API -#include "redismodule.h" +/* define macros for having usleep */ +#define _BSD_SOURCE +#define _DEFAULT_SOURCE + +#include "redismodule.h" #include #include #include diff --git a/tests/modules/misc.c b/tests/modules/misc.c new file mode 100644 index 000000000..fd892f52c --- /dev/null +++ b/tests/modules/misc.c @@ -0,0 +1,55 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include +#include + +int test_call_generic(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc<2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + const char* cmdname = RedisModule_StringPtrLen(argv[1], NULL); + RedisModuleCallReply *reply = RedisModule_Call(ctx, cmdname, "v", argv+2, argc-2); + if (reply) { + RedisModule_ReplyWithCallReply(ctx, reply); + RedisModule_FreeCallReply(reply); + } else { + RedisModule_ReplyWithError(ctx, strerror(errno)); + } + return REDISMODULE_OK; +} + +int test_call_info(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + RedisModuleCallReply *reply; + if (argc>1) + reply = RedisModule_Call(ctx, "info", "s", argv[1]); + else + reply = RedisModule_Call(ctx, "info", ""); + if (reply) { + RedisModule_ReplyWithCallReply(ctx, reply); + RedisModule_FreeCallReply(reply); + } else { + RedisModule_ReplyWithError(ctx, strerror(errno)); + } + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + if (RedisModule_Init(ctx,"misc",1,REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"test.call_generic", test_call_generic,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.call_info", test_call_info,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl new file mode 100644 index 000000000..d392aeab0 --- /dev/null +++ b/tests/unit/moduleapi/misc.tcl @@ -0,0 +1,19 @@ +set testmodule [file normalize tests/modules/misc.so] + + +start_server {tags {"modules"}} { + r module load $testmodule + + test {test RM_Call} { + set info [r test.call_info commandstats] + # cmdstat is not in a default section, so we also test an argument was passed + assert { [string match "*cmdstat_module*" $info] } + } + + test {test RM_Call args array} { + set info [r test.call_generic info commandstats] + # cmdstat is not in a default section, so we also test an argument was passed + assert { [string match "*cmdstat_module*" $info] } + } + +} From 7be7591b7b80429394795bcb65109818081721c2 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Mon, 28 Oct 2019 17:58:07 +0530 Subject: [PATCH 227/320] Fix compilation error introduced by 5d01f01be Need to add calls to REDISMODULE_API_FUNC... --- src/redismodule.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/redismodule.h b/src/redismodule.h index 434849c77..0a0573ce9 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -461,6 +461,7 @@ void REDISMODULE_API_FUNC(RedisModule_RetainString)(RedisModuleCtx *ctx, RedisMo int REDISMODULE_API_FUNC(RedisModule_StringCompare)(RedisModuleString *a, RedisModuleString *b); RedisModuleCtx *REDISMODULE_API_FUNC(RedisModule_GetContextFromIO)(RedisModuleIO *io); const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromIO)(RedisModuleIO *io); +const RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetKeyNameFromModuleKey)(RedisModuleKey *key); long long REDISMODULE_API_FUNC(RedisModule_Milliseconds)(void); void REDISMODULE_API_FUNC(RedisModule_DigestAddStringBuffer)(RedisModuleDigest *md, unsigned char *ele, size_t len); void REDISMODULE_API_FUNC(RedisModule_DigestAddLongLong)(RedisModuleDigest *md, long long ele); @@ -513,6 +514,8 @@ void REDISMODULE_API_FUNC(RedisModule_FreeThreadSafeContext)(RedisModuleCtx *ctx void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextLock)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_ThreadSafeContextUnlock)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_SubscribeToKeyspaceEvents)(RedisModuleCtx *ctx, int types, RedisModuleNotificationFunc cb); +int REDISMODULE_API_FUNC(RedisModule_NotifyKeyspaceEvent)(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key); +int REDISMODULE_API_FUNC(RedisModule_GetNotifyKeyspaceEvents)(); int REDISMODULE_API_FUNC(RedisModule_BlockedClientDisconnected)(RedisModuleCtx *ctx); void REDISMODULE_API_FUNC(RedisModule_RegisterClusterMessageReceiver)(RedisModuleCtx *ctx, uint8_t type, RedisModuleClusterMessageReceiver callback); int REDISMODULE_API_FUNC(RedisModule_SendClusterMessage)(RedisModuleCtx *ctx, char *target_id, uint8_t type, unsigned char *msg, uint32_t len); From 57649ce0519c5b85b0dae5c9d60e8655b602e64a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 23 Oct 2019 11:53:15 +0300 Subject: [PATCH 228/320] Module API for controlling LRU and LFU, and OpenKey without TOUCH Some commands would want to open a key without touching it's LRU/LFU similarly to the OBJECT or DEBUG command do. Other commands may want to implement logic similar to what RESTORE does (and in the future MIGRATE) and get/set the LRU or LFU. --- src/db.c | 8 ++++++-- src/module.c | 39 +++++++++++++++++++++++++++++++++++++-- src/object.c | 5 ++++- src/redismodule.h | 8 ++++++++ src/server.h | 3 ++- 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/db.c b/src/db.c index 2c0a0cdd3..14f163a8f 100644 --- a/src/db.c +++ b/src/db.c @@ -151,9 +151,13 @@ robj *lookupKeyRead(redisDb *db, robj *key) { * * Returns the linked value object if the key exists or NULL if the key * does not exist in the specified DB. */ -robj *lookupKeyWrite(redisDb *db, robj *key) { +robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { expireIfNeeded(db,key); - return lookupKey(db,key,LOOKUP_NONE); + return lookupKey(db,key,flags); +} + +robj *lookupKeyWrite(redisDb *db, robj *key) { + return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply) { diff --git a/src/module.c b/src/module.c index 2a1bda879..09cf6ac7a 100644 --- a/src/module.c +++ b/src/module.c @@ -1830,11 +1830,12 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { RedisModuleKey *kp; robj *value; + int flags = mode & REDISMODULE_OPEN_KEY_NOTOUCH? LOOKUP_NOTOUCH: 0; if (mode & REDISMODULE_WRITE) { - value = lookupKeyWrite(ctx->client->db,keyname); + value = lookupKeyWriteWithFlags(ctx->client->db,keyname, flags); } else { - value = lookupKeyRead(ctx->client->db,keyname); + value = lookupKeyReadWithFlags(ctx->client->db,keyname, flags); if (value == NULL) { return NULL; } @@ -6397,6 +6398,38 @@ size_t moduleCount(void) { return dictSize(modules); } +/* Set the key LRU/LFU depending on server.maxmemory_policy. + * The lru_idle arg is idle time in seconds, and is only relevant if the + * eviction policy is LRU based. + * The lfu_freq arg is a logarithmic counter that provides an indication of + * the access frequencyonly (must be <= 255) and is only relevant if the + * eviction policy is LFU based. + * Either or both of them may be <0, in that case, nothing is set. */ +/* return value is an indication if the lru field was updated or not. */ +int RM_SetLRUOrLFU(RedisModuleKey *key, long long lfu_freq, long long lru_idle) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, lfu_freq, lru_idle, lru_idle>=0 ? LRU_CLOCK() : 0)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key LRU or LFU (depending on the current eviction policy). + * One will be set to the appropiate return value, and the other will be set to -1. + * see RedisModule_SetLRUOrLFU for units and ranges. + * return value is an indication of success. */ +int RM_GetLRUOrLFU(RedisModuleKey *key, long long *lfu_freq, long long *lru_idle) { + *lru_idle = *lfu_freq = -1; + if (!key->value) + return REDISMODULE_ERR; + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { + *lfu_freq = LFUDecrAndReturn(key->value); + } else { + *lru_idle = estimateObjectIdleTime(key->value)/1000; + } + return REDISMODULE_OK; +} + /* Register all the APIs we export. Keep this function at the end of the * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { @@ -6589,4 +6622,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldULongLong); REGISTER_API(GetClientInfoById); REGISTER_API(SubscribeToServerEvent); + REGISTER_API(SetLRUOrLFU); + REGISTER_API(GetLRUOrLFU); } diff --git a/src/object.c b/src/object.c index 70022f897..47b21ae1f 100644 --- a/src/object.c +++ b/src/object.c @@ -1209,12 +1209,13 @@ sds getMemoryDoctorReport(void) { * The lru_idle and lru_clock args are only relevant if policy * is MAXMEMORY_FLAG_LRU. * Either or both of them may be <0, in that case, nothing is set. */ -void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, +int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, long long lru_clock) { if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { if (lfu_freq >= 0) { serverAssert(lfu_freq <= 255); val->lru = (LFUGetTimeInMinutes()<<8) | lfu_freq; + return 1; } } else if (lru_idle >= 0) { /* Provided LRU idle time is in seconds. Scale @@ -1231,7 +1232,9 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, if (lru_abs < 0) lru_abs = (lru_clock+(LRU_CLOCK_MAX/2)) % LRU_CLOCK_MAX; val->lru = lru_abs; + return 1; } + return 0; } /* ======================= The OBJECT and MEMORY commands =================== */ diff --git a/src/redismodule.h b/src/redismodule.h index 7053840b2..96e9fb4fa 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -18,6 +18,10 @@ #define REDISMODULE_READ (1<<0) #define REDISMODULE_WRITE (1<<1) +/* RedisModule_OpenKey extra flags for the 'mode' argument. + * Avoid touching the LRU/LFU of the key when opened. */ +#define REDISMODULE_OPEN_KEY_NOTOUCH (1<<16) + #define REDISMODULE_LIST_HEAD 0 #define REDISMODULE_LIST_TAIL 1 @@ -503,6 +507,8 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); +int REDISMODULE_API_FUNC(RedisModule_SetLRUOrLFU)(RedisModuleKey *key, long long lfu_freq, long long lru_idle); +int REDISMODULE_API_FUNC(RedisModule_GetLRUOrLFU)(RedisModuleKey *key, long long *lfu_freq, long long *lru_idle); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -703,6 +709,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldULongLong); REDISMODULE_GET_API(GetClientInfoById); REDISMODULE_GET_API(SubscribeToServerEvent); + REDISMODULE_GET_API(SetLRUOrLFU); + REDISMODULE_GET_API(GetLRUOrLFU); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/src/server.h b/src/server.h index 97672d727..30b25a918 100644 --- a/src/server.h +++ b/src/server.h @@ -2079,9 +2079,10 @@ robj *lookupKeyWrite(redisDb *db, robj *key); robj *lookupKeyReadOrReply(client *c, robj *key, robj *reply); robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply); robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags); +robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags); robj *objectCommandLookup(client *c, robj *key); robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); -void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, +int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, long long lru_clock); #define LOOKUP_NONE 0 #define LOOKUP_NOTOUCH (1<<0) From 5752b1718b3b729971eb3db5d363d8537caa65da Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 29 Oct 2019 17:29:06 +0200 Subject: [PATCH 229/320] test infra: improve prints on failed assertions sometimes we have several assertions with the same condition in the same test at different stages, and when these fail (the ones that print the condition text) you don't know which one it was. other assertions didn't print the condition text (variable names), just the expected and unexpected values. So now, all assertions print context line, and conditin text. besides, one of the major differences between 'assert' and 'assert_equal', is that the later is able to print the value that doesn't match the expected. if there is a rare non-reproducible failure, it is helpful to know what was the value the test encountered and how far it was from the threshold. So now, adding assert_lessthan and assert_range that can be used in some places. were we used just 'assert { a > b }' so far. --- tests/support/test.tcl | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 2646acecd..5e8916236 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -11,28 +11,55 @@ proc fail {msg} { proc assert {condition} { if {![uplevel 1 [list expr $condition]]} { - error "assertion:Expected condition '$condition' to be true ([uplevel 1 [list subst -nocommands $condition]])" + set context "(context: [info frame -1])" + error "assertion:Expected [uplevel 1 [list subst -nocommands $condition]] $context" } } proc assert_no_match {pattern value} { if {[string match $pattern $value]} { - error "assertion:Expected '$value' to not match '$pattern'" + set context "(context: [info frame -1])" + error "assertion:Expected '$value' to not match '$pattern' $context" } } proc assert_match {pattern value} { if {![string match $pattern $value]} { - error "assertion:Expected '$value' to match '$pattern'" + set context "(context: [info frame -1])" + error "assertion:Expected '$value' to match '$pattern' $context" } } -proc assert_equal {expected value {detail ""}} { +proc assert_equal {value expected {detail ""}} { if {$expected ne $value} { if {$detail ne ""} { - set detail " (detail: $detail)" + set detail "(detail: $detail)" + } else { + set detail "(context: [info frame -1])" } - error "assertion:Expected '$value' to be equal to '$expected'$detail" + error "assertion:Expected '$value' to be equal to '$expected' $detail" + } +} + +proc assert_lessthan {value expected {detail ""}} { + if {!($value < $expected)} { + if {$detail ne ""} { + set detail "(detail: $detail)" + } else { + set detail "(context: [info frame -1])" + } + error "assertion:Expected '$value' to be lessthan to '$expected' $detail" + } +} + +proc assert_range {value min max {detail ""}} { + if {!($value <= $max && $value >= $min)} { + if {$detail ne ""} { + set detail "(detail: $detail)" + } else { + set detail "(context: [info frame -1])" + } + error "assertion:Expected '$value' to be between to '$min' and '$max' $detail" } } From 68c6aacf3b956f64836feacdf69e8dc8ca901d19 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 29 Oct 2019 17:59:09 +0200 Subject: [PATCH 230/320] Modules hooks: complete missing hooks for the initial set of hooks * replication hooks: role change, master link status, replica online/offline * persistence hooks: saving, loading, loading progress * misc hooks: cron loop, shutdown, module loaded/unloaded * change the way hooks test work, and add tests for all of the above startLoading() now gets flag indicating what is loaded. stopLoading() now gets an indication of success or failure. adding startSaving() and stopSaving() with similar args and role. --- src/aof.c | 14 +- src/debug.c | 2 +- src/module.c | 161 +++++++++++++++++---- src/networking.c | 5 + src/rdb.c | 80 ++++++++--- src/rdb.h | 11 +- src/redis-check-rdb.c | 6 +- src/redismodule.h | 86 +++++++++-- src/replication.c | 55 ++++++- src/server.c | 11 +- src/server.h | 10 +- tests/modules/hooks.c | 256 +++++++++++++++++++++++++++++++-- tests/unit/moduleapi/hooks.tcl | 134 +++++++++++++++-- 13 files changed, 737 insertions(+), 94 deletions(-) diff --git a/src/aof.c b/src/aof.c index 0e3648ff0..dda9579e3 100644 --- a/src/aof.c +++ b/src/aof.c @@ -731,7 +731,7 @@ int loadAppendOnlyFile(char *filename) { server.aof_state = AOF_OFF; fakeClient = createAOFClient(); - startLoadingFile(fp, filename); + startLoadingFile(fp, filename, RDBFLAGS_AOF_PREAMBLE); /* Check if this AOF file has an RDB preamble. In that case we need to * load the RDB file and later continue loading the AOF tail. */ @@ -746,7 +746,7 @@ int loadAppendOnlyFile(char *filename) { serverLog(LL_NOTICE,"Reading RDB preamble from AOF file..."); if (fseek(fp,0,SEEK_SET) == -1) goto readerr; rioInitWithFile(&rdb,fp); - if (rdbLoadRio(&rdb,NULL,1) != C_OK) { + if (rdbLoadRio(&rdb,RDBFLAGS_AOF_PREAMBLE,NULL) != C_OK) { serverLog(LL_WARNING,"Error reading the RDB preamble of the AOF file, AOF loading aborted"); goto readerr; } else { @@ -767,6 +767,7 @@ int loadAppendOnlyFile(char *filename) { if (!(loops++ % 1000)) { loadingProgress(ftello(fp)); processEventsWhileBlocked(); + processModuleLoadingProgressEvent(1); } if (fgets(buf,sizeof(buf),fp) == NULL) { @@ -859,7 +860,7 @@ loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ fclose(fp); freeFakeClient(fakeClient); server.aof_state = old_aof_state; - stopLoading(); + stopLoading(1); aofUpdateCurrentSize(); server.aof_rewrite_base_size = server.aof_current_size; server.aof_fsync_offset = server.aof_current_size; @@ -1400,9 +1401,11 @@ int rewriteAppendOnlyFile(char *filename) { if (server.aof_rewrite_incremental_fsync) rioSetAutoSync(&aof,REDIS_AUTOSYNC_BYTES); + startSaving(RDBFLAGS_AOF_PREAMBLE); + if (server.aof_use_rdb_preamble) { int error; - if (rdbSaveRio(&aof,&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) { + if (rdbSaveRio(&aof,&error,RDBFLAGS_AOF_PREAMBLE,NULL) == C_ERR) { errno = error; goto werr; } @@ -1465,15 +1468,18 @@ int rewriteAppendOnlyFile(char *filename) { if (rename(tmpfile,filename) == -1) { serverLog(LL_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno)); unlink(tmpfile); + stopSaving(0); return C_ERR; } serverLog(LL_NOTICE,"SYNC append only file rewrite performed"); + stopSaving(1); return C_OK; werr: serverLog(LL_WARNING,"Write error writing append only file on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); + stopSaving(0); return C_ERR; } diff --git a/src/debug.c b/src/debug.c index 179f6d2c9..a2d37337d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -417,7 +417,7 @@ NULL } emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); protectClient(c); - int ret = rdbLoad(server.rdb_filename,NULL); + int ret = rdbLoad(server.rdb_filename,NULL,RDBFLAGS_NONE); unprotectClient(c); if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); diff --git a/src/module.c b/src/module.c index 2a1bda879..57e388881 100644 --- a/src/module.c +++ b/src/module.c @@ -1620,6 +1620,27 @@ int modulePopulateClientInfoStructure(void *ci, client *client, int structver) { return REDISMODULE_OK; } +/* This is an helper for moduleFireServerEvent() and other functions: + * It populates the replication info structure with the appropriate + * fields depending on the version provided. If the version is not valid + * then REDISMODULE_ERR is returned. Otherwise the function returns + * REDISMODULE_OK and the structure pointed by 'ri' gets populated. */ +int modulePopulateReplicationInfoStructure(void *ri, int structver) { + if (structver != 1) return REDISMODULE_ERR; + + RedisModuleReplicationInfoV1 *ri1 = ri; + memset(ri1,0,sizeof(*ri1)); + ri1->version = structver; + ri1->master = server.masterhost==NULL; + ri1->masterhost = server.masterhost? server.masterhost: ""; + ri1->masterport = server.masterport; + ri1->replid1 = server.replid; + ri1->replid2 = server.replid2; + ri1->repl1_offset = server.master_repl_offset; + ri1->repl2_offset = server.second_replid_offset; + return REDISMODULE_OK; +} + /* Return information about the client with the specified ID (that was * previously obtained via the RedisModule_GetClientId() API). If the * client exists, REDISMODULE_OK is returned, otherwise REDISMODULE_ERR @@ -5780,8 +5801,8 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * * The following sub events are available: * - * REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER - * REDISMODULE_EVENT_REPLROLECHANGED_NOW_REPLICA + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_MASTER + * REDISMODULE_SUBEVENT_REPLROLECHANGED_NOW_REPLICA * * The 'data' field can be casted by the callback to a * RedisModuleReplicationInfo structure with the following fields: @@ -5791,24 +5812,30 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * int masterport; // master instance port for NOW_REPLICA * char *replid1; // Main replication ID * char *replid2; // Secondary replication ID + * uint64_t repl1_offset; // Main replication offset * uint64_t repl2_offset; // Offset of replid2 validity - * uint64_t main_repl_offset; // Replication offset * * RedisModuleEvent_Persistence * * This event is called when RDB saving or AOF rewriting starts * and ends. The following sub events are available: * - * REDISMODULE_EVENT_LOADING_RDB_START // BGSAVE start - * REDISMODULE_EVENT_LOADING_RDB_END // BGSAVE end - * REDISMODULE_EVENT_LOADING_SYNC_RDB_START // SAVE start - * REDISMODULE_EVENT_LOADING_SYNC_RDB_START // SAVE end - * REDISMODULE_EVENT_LOADING_AOF_START // AOF rewrite start - * REDISMODULE_EVENT_LOADING_AOF_END // AOF rewrite end + * REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START + * REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START + * REDISMODULE_SUBEVENT_PERSISTENCE_ENDED + * REDISMODULE_SUBEVENT_PERSISTENCE_FAILED * * The above events are triggered not just when the user calls the * relevant commands like BGSAVE, but also when a saving operation * or AOF rewriting occurs because of internal server triggers. + * The SYNC_RDB_START sub events are happening in the forground due to + * SAVE command, FLUSHALL, or server shutdown, and the other RDB and + * AOF sub events are executed in a background fork child, so any + * action the module takes can only affect the generated AOF or RDB, + * but will not be reflected in the parent process and affect connected + * clients and commands. Also note that the AOF_START sub event may end + * up saving RDB content in case of an AOF with rdb-preamble. * * RedisModuleEvent_FlushDB * @@ -5816,8 +5843,8 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * because of replication, after the replica synchronization) * happened. The following sub events are available: * - * REDISMODULE_EVENT_FLUSHDB_START - * REDISMODULE_EVENT_FLUSHDB_END + * REDISMODULE_SUBEVENT_FLUSHDB_START + * REDISMODULE_SUBEVENT_FLUSHDB_END * * The data pointer can be casted to a RedisModuleFlushInfo * structure with the following fields: @@ -5841,12 +5868,15 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * replica is loading the RDB file from the master. * The following sub events are available: * - * REDISMODULE_EVENT_LOADING_RDB_START - * REDISMODULE_EVENT_LOADING_RDB_END - * REDISMODULE_EVENT_LOADING_MASTER_RDB_START - * REDISMODULE_EVENT_LOADING_MASTER_RDB_END - * REDISMODULE_EVENT_LOADING_AOF_START - * REDISMODULE_EVENT_LOADING_AOF_END + * REDISMODULE_SUBEVENT_LOADING_RDB_START + * REDISMODULE_SUBEVENT_LOADING_AOF_START + * REDISMODULE_SUBEVENT_LOADING_REPL_START + * REDISMODULE_SUBEVENT_LOADING_ENDED + * REDISMODULE_SUBEVENT_LOADING_FAILED + * + * Note that AOF loading may start with an RDB data in case of + * rdb-preamble, in which case you'll only recieve an AOF_START event. + * * * RedisModuleEvent_ClientChange * @@ -5855,8 +5885,8 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * structure, documented in RedisModule_GetClientInfoById(). * The following sub events are available: * - * REDISMODULE_EVENT_CLIENT_CHANGE_CONNECTED - * REDISMODULE_EVENT_CLIENT_CHANGE_DISCONNECTED + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED + * REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED * * RedisModuleEvent_Shutdown * @@ -5869,8 +5899,8 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * replica since it gets disconnected. * The following sub events are availble: * - * REDISMODULE_EVENT_REPLICA_CHANGE_ONLINE - * REDISMODULE_EVENT_REPLICA_CHANGE_OFFLINE + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE + * REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE * * No additional information is available so far: future versions * of Redis will have an API in order to enumerate the replicas @@ -5885,6 +5915,11 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * this changes depending on the "hz" configuration. * No sub events are available. * + * The data pointer can be casted to a RedisModuleCronLoop + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * * RedisModuleEvent_MasterLinkChange * * This is called for replicas in order to notify when the @@ -5894,8 +5929,38 @@ void ModuleForkDoneHandler(int exitcode, int bysignal) { * replication is happening correctly. * The following sub events are available: * - * REDISMODULE_EVENT_MASTER_LINK_UP - * REDISMODULE_EVENT_MASTER_LINK_DOWN + * REDISMODULE_SUBEVENT_MASTER_LINK_UP + * REDISMODULE_SUBEVENT_MASTER_LINK_DOWN + * + * RedisModuleEvent_ModuleChange + * + * This event is called when a new module is loaded or one is unloaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_MODULE_LOADED + * REDISMODULE_SUBEVENT_MODULE_UNLOADED + * + * The data pointer can be casted to a RedisModuleModuleChange + * structure with the following fields: + * + * const char* module_name; // Name of module loaded or unloaded. + * int32_t module_version; // Module version. + * + * RedisModuleEvent_LoadingProgress + * + * This event is called repeatedly called while an RDB or AOF file + * is being loaded. + * The following sub events are availble: + * + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB + * REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF + * + * The data pointer can be casted to a RedisModuleLoadingProgress + * structure with the following fields: + * + * int32_t hz; // Approximate number of events per second. + * int32_t progress; // Approximate progress between 0 and 1024, + * or -1 if unknown. * * The function returns REDISMODULE_OK if the module was successfully subscrived * for the specified event. If the API is called from a wrong context then @@ -5954,7 +6019,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { listRewind(RedisModule_EventListeners,&li); while((ln = listNext(&li))) { RedisModuleEventListener *el = ln->value; - if (el->event.id == eid && !el->module->in_hook) { + if (el->event.id == eid) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.module = el->module; @@ -5968,6 +6033,8 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { void *moduledata = NULL; RedisModuleClientInfoV1 civ1; + RedisModuleReplicationInfoV1 riv1; + RedisModuleModuleChangeV1 mcv1; /* Start at DB zero by default when calling the handler. It's * up to the specific event setup to change it when it makes * sense. For instance for FLUSHDB events we select the correct @@ -5979,11 +6046,26 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data) { modulePopulateClientInfoStructure(&civ1,data, el->event.dataver); moduledata = &civ1; + } else if (eid == REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED) { + modulePopulateReplicationInfoStructure(&riv1,el->event.dataver); + moduledata = &riv1; } else if (eid == REDISMODULE_EVENT_FLUSHDB) { moduledata = data; RedisModuleFlushInfoV1 *fi = data; if (fi->dbnum != -1) selectDb(ctx.client, fi->dbnum); + } else if (eid == REDISMODULE_EVENT_MODULE_CHANGE) { + RedisModule *m = data; + if (m == el->module) + continue; + mcv1.version = REDISMODULE_MODULE_CHANGE_VERSION; + mcv1.module_name = m->name; + mcv1.module_version = m->ver; + moduledata = &mcv1; + } else if (eid == REDISMODULE_EVENT_LOADING_PROGRESS) { + moduledata = data; + } else if (eid == REDISMODULE_EVENT_CRON_LOOP) { + moduledata = data; } ModulesInHooks++; @@ -6015,6 +6097,27 @@ void moduleUnsubscribeAllServerEvents(RedisModule *module) { } } +void processModuleLoadingProgressEvent(int is_aof) { + long long now = ustime(); + static long long next_event = 0; + if (now >= next_event) { + /* Fire the loading progress modules end event. */ + int progress = -1; + if (server.loading_total_bytes) + progress = (server.loading_total_bytes<<10) / server.loading_total_bytes; + RedisModuleFlushInfoV1 fi = {REDISMODULE_LOADING_PROGRESS_VERSION, + server.hz, + progress}; + moduleFireServerEvent(REDISMODULE_EVENT_LOADING_PROGRESS, + is_aof? + REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF: + REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB, + &fi); + /* decide when the next event should fire. */ + next_event = now + 1000000 / server.hz; + } +} + /* -------------------------------------------------------------------------- * Modules API internals * -------------------------------------------------------------------------- */ @@ -6183,6 +6286,11 @@ int moduleLoad(const char *path, void **module_argv, int module_argc) { ctx.module->blocked_clients = 0; ctx.module->handle = handle; serverLog(LL_NOTICE,"Module '%s' loaded from %s",ctx.module->name,path); + /* Fire the loaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_LOADED, + ctx.module); + moduleFreeContext(&ctx); return C_OK; } @@ -6245,6 +6353,11 @@ int moduleUnload(sds name) { module->name, error); } + /* Fire the unloaded modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MODULE_CHANGE, + REDISMODULE_SUBEVENT_MODULE_UNLOADED, + module); + /* Remove from list of modules. */ serverLog(LL_NOTICE,"Module %s unloaded",module->name); dictDelete(modules,module->name); diff --git a/src/networking.c b/src/networking.c index e7cc561fa..9336c177c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1118,6 +1118,11 @@ void freeClient(client *c) { if (c->flags & CLIENT_SLAVE && listLength(server.slaves) == 0) server.repl_no_slaves_since = server.unixtime; refreshGoodSlavesCount(); + /* Fire the replica change modules event. */ + if (c->replstate == SLAVE_STATE_ONLINE) + moduleFireServerEvent(REDISMODULE_EVENT_REPLICA_CHANGE, + REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE, + NULL); } /* Master/slave cleanup Case 2: diff --git a/src/rdb.c b/src/rdb.c index f530219a4..b569edfea 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1080,9 +1080,9 @@ ssize_t rdbSaveAuxFieldStrInt(rio *rdb, char *key, long long val) { } /* Save a few default AUX fields with information about the RDB generated. */ -int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) { +int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { int redis_bits = (sizeof(void*) == 8) ? 64 : 32; - int aof_preamble = (flags & RDB_SAVE_AOF_PREAMBLE) != 0; + int aof_preamble = (rdbflags & RDBFLAGS_AOF_PREAMBLE) != 0; /* Add a few fields about the state when the RDB was created. */ if (rdbSaveAuxFieldStrStr(rdb,"redis-ver",REDIS_VERSION) == -1) return -1; @@ -1150,7 +1150,7 @@ ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt) { * When the function returns C_ERR and if 'error' is not NULL, the * integer pointed by 'error' is set to the value of errno just after the I/O * error. */ -int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { +int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi) { dictIterator *di = NULL; dictEntry *de; char magic[10]; @@ -1162,7 +1162,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { rdb->update_cksum = rioGenericUpdateChecksum; snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION); if (rdbWriteRaw(rdb,magic,9) == -1) goto werr; - if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr; + if (rdbSaveInfoAuxFields(rdb,rdbflags,rsi) == -1) goto werr; if (rdbSaveModulesAux(rdb, REDISMODULE_AUX_BEFORE_RDB) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { @@ -1199,7 +1199,7 @@ int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) { /* When this RDB is produced as part of an AOF rewrite, move * accumulated diff from parent to child while rewriting in * order to have a smaller final write. */ - if (flags & RDB_SAVE_AOF_PREAMBLE && + if (rdbflags & RDBFLAGS_AOF_PREAMBLE && rdb->processed_bytes > processed+AOF_READ_DIFF_INTERVAL_BYTES) { processed = rdb->processed_bytes; @@ -1254,18 +1254,21 @@ werr: int rdbSaveRioWithEOFMark(rio *rdb, int *error, rdbSaveInfo *rsi) { char eofmark[RDB_EOF_MARK_SIZE]; + startSaving(RDBFLAGS_REPLICATION); getRandomHexChars(eofmark,RDB_EOF_MARK_SIZE); if (error) *error = 0; if (rioWrite(rdb,"$EOF:",5) == 0) goto werr; if (rioWrite(rdb,eofmark,RDB_EOF_MARK_SIZE) == 0) goto werr; if (rioWrite(rdb,"\r\n",2) == 0) goto werr; - if (rdbSaveRio(rdb,error,RDB_SAVE_NONE,rsi) == C_ERR) goto werr; + if (rdbSaveRio(rdb,error,RDBFLAGS_NONE,rsi) == C_ERR) goto werr; if (rioWrite(rdb,eofmark,RDB_EOF_MARK_SIZE) == 0) goto werr; + stopSaving(1); return C_OK; werr: /* Write error. */ /* Set 'error' only if not already set by rdbSaveRio() call. */ if (error && *error == 0) *error = errno; + stopSaving(0); return C_ERR; } @@ -1291,11 +1294,12 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) { } rioInitWithFile(&rdb,fp); + startSaving(RDBFLAGS_NONE); if (server.rdb_save_incremental_fsync) rioSetAutoSync(&rdb,REDIS_AUTOSYNC_BYTES); - if (rdbSaveRio(&rdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) { + if (rdbSaveRio(&rdb,&error,RDBFLAGS_NONE,rsi) == C_ERR) { errno = error; goto werr; } @@ -1317,6 +1321,7 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) { cwdp ? cwdp : "unknown", strerror(errno)); unlink(tmpfile); + stopSaving(0); return C_ERR; } @@ -1324,12 +1329,14 @@ int rdbSave(char *filename, rdbSaveInfo *rsi) { server.dirty = 0; server.lastsave = time(NULL); server.lastbgsave_status = C_OK; + stopSaving(1); return C_OK; werr: serverLog(LL_WARNING,"Write error saving DB on disk: %s", strerror(errno)); fclose(fp); unlink(tmpfile); + stopSaving(0); return C_ERR; } @@ -1918,23 +1925,33 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, robj *key) { /* Mark that we are loading in the global state and setup the fields * needed to provide loading stats. */ -void startLoading(size_t size) { +void startLoading(size_t size, int rdbflags) { /* Load the DB */ server.loading = 1; server.loading_start_time = time(NULL); server.loading_loaded_bytes = 0; server.loading_total_bytes = size; + + /* Fire the loading modules start event. */ + int subevent; + if (rdbflags & RDBFLAGS_AOF_PREAMBLE) + subevent = REDISMODULE_SUBEVENT_LOADING_AOF_START; + else if(rdbflags & RDBFLAGS_REPLICATION) + subevent = REDISMODULE_SUBEVENT_LOADING_REPL_START; + else + subevent = REDISMODULE_SUBEVENT_LOADING_RDB_START; + moduleFireServerEvent(REDISMODULE_EVENT_LOADING,subevent,NULL); } /* Mark that we are loading in the global state and setup the fields * needed to provide loading stats. * 'filename' is optional and used for rdb-check on error */ -void startLoadingFile(FILE *fp, char* filename) { +void startLoadingFile(FILE *fp, char* filename, int rdbflags) { struct stat sb; if (fstat(fileno(fp), &sb) == -1) sb.st_size = 0; rdbFileBeingLoaded = filename; - startLoading(sb.st_size); + startLoading(sb.st_size, rdbflags); } /* Refresh the loading progress info */ @@ -1945,9 +1962,37 @@ void loadingProgress(off_t pos) { } /* Loading finished */ -void stopLoading(void) { +void stopLoading(int success) { server.loading = 0; rdbFileBeingLoaded = NULL; + + /* Fire the loading modules end event. */ + moduleFireServerEvent(REDISMODULE_EVENT_LOADING, + success? + REDISMODULE_SUBEVENT_LOADING_ENDED: + REDISMODULE_SUBEVENT_LOADING_FAILED, + NULL); +} + +void startSaving(int rdbflags) { + /* Fire the persistence modules end event. */ + int subevent; + if (rdbflags & RDBFLAGS_AOF_PREAMBLE) + subevent = REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START; + else if (getpid()!=server.pid) + subevent = REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START; + else + subevent = REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START; + moduleFireServerEvent(REDISMODULE_EVENT_PERSISTENCE,subevent,NULL); +} + +void stopSaving(int success) { + /* Fire the persistence modules end event. */ + moduleFireServerEvent(REDISMODULE_EVENT_PERSISTENCE, + success? + REDISMODULE_SUBEVENT_PERSISTENCE_ENDED: + REDISMODULE_SUBEVENT_PERSISTENCE_FAILED, + NULL); } /* Track loading progress in order to serve client's from time to time @@ -1966,12 +2011,13 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { replicationSendNewlineToMaster(); loadingProgress(r->processed_bytes); processEventsWhileBlocked(); + processModuleLoadingProgressEvent(0); } } /* Load an RDB file from the rio stream 'rdb'. On success C_OK is returned, * otherwise C_ERR is returned and 'errno' is set accordingly. */ -int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { +int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { uint64_t dbid; int type, rdbver; redisDb *db = server.db+0; @@ -2182,7 +2228,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) { * received from the master. In the latter case, the master is * responsible for key expiry. If we would expire keys here, the * snapshot taken by the master may not be reflected on the slave. */ - if (server.masterhost == NULL && !loading_aof && expiretime != -1 && expiretime < now) { + if (server.masterhost == NULL && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { decrRefCount(key); decrRefCount(val); } else { @@ -2243,17 +2289,17 @@ eoferr: * * If you pass an 'rsi' structure initialied with RDB_SAVE_OPTION_INIT, the * loading code will fiil the information fields in the structure. */ -int rdbLoad(char *filename, rdbSaveInfo *rsi) { +int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags) { FILE *fp; rio rdb; int retval; if ((fp = fopen(filename,"r")) == NULL) return C_ERR; - startLoadingFile(fp, filename); + startLoadingFile(fp, filename,rdbflags); rioInitWithFile(&rdb,fp); - retval = rdbLoadRio(&rdb,rsi,0); + retval = rdbLoadRio(&rdb,rdbflags,rsi); fclose(fp); - stopLoading(); + stopLoading(retval==C_OK); return retval; } diff --git a/src/rdb.h b/src/rdb.h index 40a50f7ba..4229beea8 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -121,8 +121,10 @@ #define RDB_LOAD_PLAIN (1<<1) #define RDB_LOAD_SDS (1<<2) -#define RDB_SAVE_NONE 0 -#define RDB_SAVE_AOF_PREAMBLE (1<<0) +/* flags on the purpose of rdb save or load */ +#define RDBFLAGS_NONE 0 +#define RDBFLAGS_AOF_PREAMBLE (1<<0) +#define RDBFLAGS_REPLICATION (1<<1) int rdbSaveType(rio *rdb, unsigned char type); int rdbLoadType(rio *rdb); @@ -135,7 +137,7 @@ uint64_t rdbLoadLen(rio *rdb, int *isencoded); int rdbLoadLenByRef(rio *rdb, int *isencoded, uint64_t *lenptr); int rdbSaveObjectType(rio *rdb, robj *o); int rdbLoadObjectType(rio *rdb); -int rdbLoad(char *filename, rdbSaveInfo *rsi); +int rdbLoad(char *filename, rdbSaveInfo *rsi, int rdbflags); int rdbSaveBackground(char *filename, rdbSaveInfo *rsi); int rdbSaveToSlavesSockets(rdbSaveInfo *rsi); void rdbRemoveTempFile(pid_t childpid); @@ -154,7 +156,8 @@ int rdbSaveBinaryDoubleValue(rio *rdb, double val); int rdbLoadBinaryDoubleValue(rio *rdb, double *val); int rdbSaveBinaryFloatValue(rio *rdb, float val); int rdbLoadBinaryFloatValue(rio *rdb, float *val); -int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof); +int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi); +int rdbSaveRio(rio *rdb, int *error, int rdbflags, rdbSaveInfo *rsi); rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi); #endif diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 5e7415046..1210d49b4 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -202,7 +202,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } expiretime = -1; - startLoadingFile(fp, rdbfilename); + startLoadingFile(fp, rdbfilename, RDBFLAGS_NONE); while(1) { robj *key, *val; @@ -316,7 +316,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { } if (closefile) fclose(fp); - stopLoading(); + stopLoading(1); return 0; eoferr: /* unexpected end of file is handled here with a fatal exit */ @@ -327,7 +327,7 @@ eoferr: /* unexpected end of file is handled here with a fatal exit */ } err: if (closefile) fclose(fp); - stopLoading(); + stopLoading(0); return 1; } diff --git a/src/redismodule.h b/src/redismodule.h index 7053840b2..7ae1ad692 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -181,6 +181,8 @@ typedef uint64_t RedisModuleTimerID; #define REDISMODULE_EVENT_REPLICA_CHANGE 6 #define REDISMODULE_EVENT_MASTER_LINK_CHANGE 7 #define REDISMODULE_EVENT_CRON_LOOP 8 +#define REDISMODULE_EVENT_MODULE_CHANGE 9 +#define REDISMODULE_EVENT_LOADING_PROGRESS 10 typedef struct RedisModuleEvent { uint64_t id; /* REDISMODULE_EVENT_... defines. */ @@ -226,18 +228,28 @@ static const RedisModuleEvent RedisModuleEvent_MasterLinkChange = { REDISMODULE_EVENT_MASTER_LINK_CHANGE, 1 + }, + RedisModuleEvent_ModuleChange = { + REDISMODULE_EVENT_MODULE_CHANGE, + 1 + }, + RedisModuleEvent_LoadingProgress = { + REDISMODULE_EVENT_LOADING_PROGRESS, + 1 }; /* Those are values that are used for the 'subevent' callback argument. */ #define REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START 0 -#define REDISMODULE_SUBEVENT_PERSISTENCE_RDB_END 1 -#define REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START 2 -#define REDISMODULE_SUBEVENT_PERSISTENCE_AOF_END 3 +#define REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START 1 +#define REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START 2 +#define REDISMODULE_SUBEVENT_PERSISTENCE_ENDED 3 +#define REDISMODULE_SUBEVENT_PERSISTENCE_FAILED 4 #define REDISMODULE_SUBEVENT_LOADING_RDB_START 0 -#define REDISMODULE_SUBEVENT_LOADING_RDB_END 1 -#define REDISMODULE_SUBEVENT_LOADING_AOF_START 2 -#define REDISMODULE_SUBEVENT_LOADING_AOF_END 3 +#define REDISMODULE_SUBEVENT_LOADING_AOF_START 1 +#define REDISMODULE_SUBEVENT_LOADING_REPL_START 2 +#define REDISMODULE_SUBEVENT_LOADING_ENDED 3 +#define REDISMODULE_SUBEVENT_LOADING_FAILED 4 #define REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED 0 #define REDISMODULE_SUBEVENT_CLIENT_CHANGE_DISCONNECTED 1 @@ -245,12 +257,21 @@ static const RedisModuleEvent #define REDISMODULE_SUBEVENT_MASTER_LINK_UP 0 #define REDISMODULE_SUBEVENT_MASTER_LINK_DOWN 1 -#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_CONNECTED 0 -#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_DISCONNECTED 1 +#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE 0 +#define REDISMODULE_SUBEVENT_REPLICA_CHANGE_OFFLINE 1 + +#define REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER 0 +#define REDISMODULE_EVENT_REPLROLECHANGED_NOW_REPLICA 1 #define REDISMODULE_SUBEVENT_FLUSHDB_START 0 #define REDISMODULE_SUBEVENT_FLUSHDB_END 1 +#define REDISMODULE_SUBEVENT_MODULE_LOADED 0 +#define REDISMODULE_SUBEVENT_MODULE_UNLOADED 1 + +#define REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB 0 +#define REDISMODULE_SUBEVENT_LOADING_PROGRESS_AOF 1 + /* RedisModuleClientInfo flags. */ #define REDISMODULE_CLIENTINFO_FLAG_SSL (1<<0) #define REDISMODULE_CLIENTINFO_FLAG_PUBSUB (1<<1) @@ -286,6 +307,22 @@ typedef struct RedisModuleClientInfo { #define RedisModuleClientInfo RedisModuleClientInfoV1 +#define REDISMODULE_REPLICATIONINFO_VERSION 1 +typedef struct RedisModuleReplicationInfo { + uint64_t version; /* Not used since this structure is never passed + from the module to the core right now. Here + for future compatibility. */ + int master; /* true if master, false if replica */ + char *masterhost; /* master instance hostname for NOW_REPLICA */ + int masterport; /* master instance port for NOW_REPLICA */ + char *replid1; /* Main replication ID */ + char *replid2; /* Secondary replication ID */ + uint64_t repl1_offset; /* Main replication offset */ + uint64_t repl2_offset; /* Offset of replid2 validity */ +} RedisModuleReplicationInfoV1; + +#define RedisModuleReplicationInfo RedisModuleReplicationInfoV1 + #define REDISMODULE_FLUSHINFO_VERSION 1 typedef struct RedisModuleFlushInfo { uint64_t version; /* Not used since this structure is never passed @@ -297,6 +334,39 @@ typedef struct RedisModuleFlushInfo { #define RedisModuleFlushInfo RedisModuleFlushInfoV1 +#define REDISMODULE_MODULE_CHANGE_VERSION 1 +typedef struct RedisModuleModuleChange { + uint64_t version; /* Not used since this structure is never passed + from the module to the core right now. Here + for future compatibility. */ + const char* module_name;/* Name of module loaded or unloaded. */ + int32_t module_version; /* Module version. */ +} RedisModuleModuleChangeV1; + +#define RedisModuleModuleChange RedisModuleModuleChangeV1 + +#define REDISMODULE_CRON_LOOP_VERSION 1 +typedef struct RedisModuleCronLoopInfo { + uint64_t version; /* Not used since this structure is never passed + from the module to the core right now. Here + for future compatibility. */ + int32_t hz; /* Approximate number of events per second. */ +} RedisModuleCronLoopV1; + +#define RedisModuleCronLoop RedisModuleCronLoopV1 + +#define REDISMODULE_LOADING_PROGRESS_VERSION 1 +typedef struct RedisModuleLoadingProgressInfo { + uint64_t version; /* Not used since this structure is never passed + from the module to the core right now. Here + for future compatibility. */ + int32_t hz; /* Approximate number of events per second. */ + int32_t progress; /* Approximate progress between 0 and 1024, or -1 + * if unknown. */ +} RedisModuleLoadingProgressV1; + +#define RedisModuleLoadingProgress RedisModuleLoadingProgressV1 + /* ------------------------- End of common defines ------------------------ */ #ifndef REDISMODULE_CORE diff --git a/src/replication.c b/src/replication.c index 4550e6a83..c9a2e0fe1 100644 --- a/src/replication.c +++ b/src/replication.c @@ -533,6 +533,12 @@ int masterTryPartialResynchronization(client *c) { * has this state from the previous connection with the master. */ refreshGoodSlavesCount(); + + /* Fire the replica change modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_REPLICA_CHANGE, + REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE, + NULL); + return C_OK; /* The caller can return, no full resync needed. */ need_full_resync: @@ -868,6 +874,10 @@ void putSlaveOnline(client *slave) { return; } refreshGoodSlavesCount(); + /* Fire the replica change modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_REPLICA_CHANGE, + REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE, + NULL); serverLog(LL_NOTICE,"Synchronization with replica %s succeeded", replicationGetSlaveName(slave)); } @@ -1542,11 +1552,11 @@ void readSyncBulkPayload(connection *conn) { * We'll restore it when the RDB is received. */ connBlock(conn); connRecvTimeout(conn, server.repl_timeout*1000); - startLoading(server.repl_transfer_size); + startLoading(server.repl_transfer_size, RDBFLAGS_REPLICATION); - if (rdbLoadRio(&rdb,&rsi,0) != C_OK) { + if (rdbLoadRio(&rdb,RDBFLAGS_REPLICATION,&rsi) != C_OK) { /* RDB loading failed. */ - stopLoading(); + stopLoading(0); serverLog(LL_WARNING, "Failed trying to load the MASTER synchronization DB " "from socket"); @@ -1567,7 +1577,7 @@ void readSyncBulkPayload(connection *conn) { * gets promoted. */ return; } - stopLoading(); + stopLoading(1); /* RDB loading succeeded if we reach this point. */ if (server.repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB) { @@ -1614,7 +1624,7 @@ void readSyncBulkPayload(connection *conn) { cancelReplicationHandshake(); return; } - if (rdbLoad(server.rdb_filename,&rsi) != C_OK) { + if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_REPLICATION) != C_OK) { serverLog(LL_WARNING, "Failed trying to load the MASTER synchronization " "DB from disk"); @@ -1636,6 +1646,11 @@ void readSyncBulkPayload(connection *conn) { server.repl_state = REPL_STATE_CONNECTED; server.repl_down_since = 0; + /* Fire the master link modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE, + REDISMODULE_SUBEVENT_MASTER_LINK_UP, + NULL); + /* After a full resynchroniziation we use the replication ID and * offset of the master. The secondary ID / offset are cleared since * we are starting a new history. */ @@ -2314,12 +2329,31 @@ void replicationSetMaster(char *ip, int port) { replicationDiscardCachedMaster(); replicationCacheMasterUsingMyself(); } + + /* Fire the role change modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, + REDISMODULE_EVENT_REPLROLECHANGED_NOW_REPLICA, + NULL); + + /* Fire the master link modules event. */ + if (server.repl_state == REPL_STATE_CONNECTED) + moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE, + REDISMODULE_SUBEVENT_MASTER_LINK_DOWN, + NULL); + server.repl_state = REPL_STATE_CONNECT; } /* Cancel replication, setting the instance as a master itself. */ void replicationUnsetMaster(void) { if (server.masterhost == NULL) return; /* Nothing to do. */ + + /* Fire the master link modules event. */ + if (server.repl_state == REPL_STATE_CONNECTED) + moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE, + REDISMODULE_SUBEVENT_MASTER_LINK_DOWN, + NULL); + sdsfree(server.masterhost); server.masterhost = NULL; /* When a slave is turned into a master, the current replication ID @@ -2348,11 +2382,22 @@ void replicationUnsetMaster(void) { * starting from now. Otherwise the backlog will be freed after a * failover if slaves do not connect immediately. */ server.repl_no_slaves_since = server.unixtime; + + /* Fire the role change modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_REPLICATION_ROLE_CHANGED, + REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER, + NULL); } /* This function is called when the slave lose the connection with the * master into an unexpected way. */ void replicationHandleMasterDisconnection(void) { + /* Fire the master link modules event. */ + if (server.repl_state == REPL_STATE_CONNECTED) + moduleFireServerEvent(REDISMODULE_EVENT_MASTER_LINK_CHANGE, + REDISMODULE_SUBEVENT_MASTER_LINK_DOWN, + NULL); + server.master = NULL; server.repl_state = REPL_STATE_CONNECT; server.repl_down_since = server.unixtime; diff --git a/src/server.c b/src/server.c index 8f165113d..f42924764 100644 --- a/src/server.c +++ b/src/server.c @@ -2056,6 +2056,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { server.rdb_bgsave_scheduled = 0; } + /* Fire the cron loop modules event. */ + RedisModuleCronLoopV1 ei = {REDISMODULE_CRON_LOOP_VERSION,server.hz}; + moduleFireServerEvent(REDISMODULE_EVENT_CRON_LOOP, + 0, + &ei); + server.cronloops++; return 1000/server.hz; } @@ -3682,6 +3688,9 @@ int prepareForShutdown(int flags) { } } + /* Fire the shutdown modules event. */ + moduleFireServerEvent(REDISMODULE_EVENT_SHUTDOWN,0,NULL); + /* Remove the pid file if possible and needed. */ if (server.daemonize || server.pidfile) { serverLog(LL_NOTICE,"Removing the pid file."); @@ -4767,7 +4776,7 @@ void loadDataFromDisk(void) { serverLog(LL_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000); } else { rdbSaveInfo rsi = RDB_SAVE_INFO_INIT; - if (rdbLoad(server.rdb_filename,&rsi) == C_OK) { + if (rdbLoad(server.rdb_filename,&rsi,RDBFLAGS_NONE) == C_OK) { serverLog(LL_NOTICE,"DB loaded from disk: %.3f seconds", (float)(ustime()-start)/1000000); diff --git a/src/server.h b/src/server.h index 97672d727..b52d57a05 100644 --- a/src/server.h +++ b/src/server.h @@ -1602,6 +1602,7 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); +void processModuleLoadingProgressEvent(int is_aof); /* Utils */ long long ustime(void); @@ -1831,10 +1832,12 @@ void rdbPipeReadHandler(struct aeEventLoop *eventLoop, int fd, void *clientData, void rdbPipeWriteHandlerConnRemoved(struct connection *conn); /* Generic persistence functions */ -void startLoadingFile(FILE* fp, char* filename); -void startLoading(size_t size); +void startLoadingFile(FILE* fp, char* filename, int rdbflags); +void startLoading(size_t size, int rdbflags); void loadingProgress(off_t pos); -void stopLoading(void); +void stopLoading(int success); +void startSaving(int rdbflags); +void stopSaving(int success); #define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ #define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ @@ -1843,7 +1846,6 @@ int writeCommandsDeniedByDiskError(void); /* RDB persistence */ #include "rdb.h" -int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi); void killRDBChild(void); /* AOF persistence */ diff --git a/tests/modules/hooks.c b/tests/modules/hooks.c index 33b690b2f..665a20481 100644 --- a/tests/modules/hooks.c +++ b/tests/modules/hooks.c @@ -30,36 +30,227 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#define REDISMODULE_EXPERIMENTAL_API #include "redismodule.h" +#include +#include + +/* We need to store events to be able to test and see what we got, and we can't + * store them in the key-space since that would mess up rdb loading (duplicates) + * and be lost of flushdb. */ +RedisModuleDict *event_log = NULL; + +typedef struct EventElement { + long count; + RedisModuleString *last_val_string; + long last_val_int; +} EventElement; + +void LogStringEvent(RedisModuleCtx *ctx, const char* keyname, const char* data) { + EventElement *event = RedisModule_DictGetC(event_log, (void*)keyname, strlen(keyname), NULL); + if (!event) { + event = RedisModule_Alloc(sizeof(EventElement)); + memset(event, 0, sizeof(EventElement)); + RedisModule_DictSetC(event_log, (void*)keyname, strlen(keyname), event); + } + if (event->last_val_string) RedisModule_FreeString(ctx, event->last_val_string); + event->last_val_string = RedisModule_CreateString(ctx, data, strlen(data)); + event->count++; +} + +void LogNumericEvent(RedisModuleCtx *ctx, const char* keyname, long data) { + REDISMODULE_NOT_USED(ctx); + EventElement *event = RedisModule_DictGetC(event_log, (void*)keyname, strlen(keyname), NULL); + if (!event) { + event = RedisModule_Alloc(sizeof(EventElement)); + memset(event, 0, sizeof(EventElement)); + RedisModule_DictSetC(event_log, (void*)keyname, strlen(keyname), event); + } + event->last_val_int = data; + event->count++; +} + +void FreeEvent(RedisModuleCtx *ctx, EventElement *event) { + if (event->last_val_string) + RedisModule_FreeString(ctx, event->last_val_string); + RedisModule_Free(event); +} + +int cmdEventCount(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + EventElement *event = RedisModule_DictGet(event_log, argv[1], NULL); + RedisModule_ReplyWithLongLong(ctx, event? event->count: 0); + return REDISMODULE_OK; +} + +int cmdEventLast(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2){ + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + EventElement *event = RedisModule_DictGet(event_log, argv[1], NULL); + if (event && event->last_val_string) + RedisModule_ReplyWithString(ctx, event->last_val_string); + else if (event) + RedisModule_ReplyWithLongLong(ctx, event->last_val_int); + else + RedisModule_ReplyWithNull(ctx); + return REDISMODULE_OK; +} + +void clearEvents(RedisModuleCtx *ctx) +{ + RedisModuleString *key; + EventElement *event; + RedisModuleDictIter *iter = RedisModule_DictIteratorStart(event_log, "^", NULL); + while((key = RedisModule_DictNext(ctx, iter, (void**)&event)) != NULL) { + event->count = 0; + event->last_val_int = 0; + if (event->last_val_string) RedisModule_FreeString(ctx, event->last_val_string); + event->last_val_string = NULL; + RedisModule_DictDel(event_log, key, NULL); + RedisModule_Free(event); + } + RedisModule_DictIteratorStop(iter); +} + +int cmdEventsClear(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argc); + REDISMODULE_NOT_USED(argv); + clearEvents(ctx); + return REDISMODULE_OK; +} /* Client state change callback. */ void clientChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) { - REDISMODULE_NOT_USED(ctx); REDISMODULE_NOT_USED(e); RedisModuleClientInfo *ci = data; char *keyname = (sub == REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED) ? - "connected" : "disconnected"; - RedisModuleCallReply *reply; - RedisModule_SelectDb(ctx,9); - reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)ci->id); - RedisModule_FreeCallReply(reply); + "client-connected" : "client-disconnected"; + LogNumericEvent(ctx, keyname, ci->id); } void flushdbCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) { - REDISMODULE_NOT_USED(ctx); REDISMODULE_NOT_USED(e); RedisModuleFlushInfo *fi = data; char *keyname = (sub == REDISMODULE_SUBEVENT_FLUSHDB_START) ? "flush-start" : "flush-end"; - RedisModuleCallReply *reply; - RedisModule_SelectDb(ctx,9); - reply = RedisModule_Call(ctx,"RPUSH","cl",keyname,(long)fi->dbnum); - RedisModule_FreeCallReply(reply); + LogNumericEvent(ctx, keyname, fi->dbnum); +} + +void roleChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + + RedisModuleReplicationInfo *ri = data; + char *keyname = (sub == REDISMODULE_EVENT_REPLROLECHANGED_NOW_MASTER) ? + "role-master" : "role-replica"; + LogStringEvent(ctx, keyname, ri->masterhost); +} + +void replicationChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + + char *keyname = (sub == REDISMODULE_SUBEVENT_REPLICA_CHANGE_ONLINE) ? + "replica-online" : "replica-offline"; + LogNumericEvent(ctx, keyname, 0); +} + +void rasterLinkChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + + char *keyname = (sub == REDISMODULE_SUBEVENT_MASTER_LINK_UP) ? + "masterlink-up" : "masterlink-down"; + LogNumericEvent(ctx, keyname, 0); +} + +void persistenceCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + + char *keyname = NULL; + switch (sub) { + case REDISMODULE_SUBEVENT_PERSISTENCE_RDB_START: keyname = "persistence-rdb-start"; break; + case REDISMODULE_SUBEVENT_PERSISTENCE_AOF_START: keyname = "persistence-aof-start"; break; + case REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START: keyname = "persistence-syncrdb-start"; break; + case REDISMODULE_SUBEVENT_PERSISTENCE_ENDED: keyname = "persistence-end"; break; + case REDISMODULE_SUBEVENT_PERSISTENCE_FAILED: keyname = "persistence-failed"; break; + } + /* modifying the keyspace from the fork child is not an option, using log instead */ + RedisModule_Log(ctx, "warning", "module-event-%s", keyname); + if (sub == REDISMODULE_SUBEVENT_PERSISTENCE_SYNC_RDB_START) + LogNumericEvent(ctx, keyname, 0); +} + +void loadingCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + + char *keyname = NULL; + switch (sub) { + case REDISMODULE_SUBEVENT_LOADING_RDB_START: keyname = "loading-rdb-start"; break; + case REDISMODULE_SUBEVENT_LOADING_AOF_START: keyname = "loading-aof-start"; break; + case REDISMODULE_SUBEVENT_LOADING_REPL_START: keyname = "loading-repl-start"; break; + case REDISMODULE_SUBEVENT_LOADING_ENDED: keyname = "loading-end"; break; + case REDISMODULE_SUBEVENT_LOADING_FAILED: keyname = "loading-failed"; break; + } + LogNumericEvent(ctx, keyname, 0); +} + +void loadingProgressCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + + RedisModuleLoadingProgress *ei = data; + char *keyname = (sub == REDISMODULE_SUBEVENT_LOADING_PROGRESS_RDB) ? + "loading-progress-rdb" : "loading-progress-aof"; + LogNumericEvent(ctx, keyname, ei->progress); +} + +void shutdownCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(data); + REDISMODULE_NOT_USED(sub); + + RedisModule_Log(ctx, "warning", "module-event-%s", "shutdown"); +} + +void cronLoopCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + REDISMODULE_NOT_USED(sub); + + RedisModuleCronLoop *ei = data; + LogNumericEvent(ctx, "cron-loop", ei->hz); +} + +void moduleChangeCallback(RedisModuleCtx *ctx, RedisModuleEvent e, uint64_t sub, void *data) +{ + REDISMODULE_NOT_USED(e); + + RedisModuleModuleChange *ei = data; + char *keyname = (sub == REDISMODULE_SUBEVENT_MODULE_LOADED) ? + "module-loaded" : "module-unloaded"; + LogStringEvent(ctx, keyname, ei->module_name); } /* This function must be present on each Redis module. It is used in order to @@ -71,9 +262,50 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_Init(ctx,"testhook",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) return REDISMODULE_ERR; + /* replication related hooks */ + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ReplicationRoleChanged, roleChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ReplicaChange, replicationChangeCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_MasterLinkChange, rasterLinkChangeCallback); + + /* persistence related hooks */ + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_Persistence, persistenceCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_Loading, loadingCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_LoadingProgress, loadingProgressCallback); + + /* other hooks */ RedisModule_SubscribeToServerEvent(ctx, RedisModuleEvent_ClientChange, clientChangeCallback); RedisModule_SubscribeToServerEvent(ctx, RedisModuleEvent_FlushDB, flushdbCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_Shutdown, shutdownCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_CronLoop, cronLoopCallback); + RedisModule_SubscribeToServerEvent(ctx, + RedisModuleEvent_ModuleChange, moduleChangeCallback); + + event_log = RedisModule_CreateDict(ctx); + + if (RedisModule_CreateCommand(ctx,"hooks.event_count", cmdEventCount,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"hooks.event_last", cmdEventLast,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"hooks.clear", cmdEventsClear,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } + +int RedisModule_OnUnload(RedisModuleCtx *ctx) { + clearEvents(ctx); + RedisModule_FreeDict(ctx, event_log); + event_log = NULL; + return REDISMODULE_OK; +} + diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl index 7a727902d..cbca9c3eb 100644 --- a/tests/unit/moduleapi/hooks.tcl +++ b/tests/unit/moduleapi/hooks.tcl @@ -3,26 +3,138 @@ set testmodule [file normalize tests/modules/hooks.so] tags "modules" { start_server {} { r module load $testmodule + r config set appendonly yes + test {Test clients connection / disconnection hooks} { for {set j 0} {$j < 2} {incr j} { set rd1 [redis_deferring_client] $rd1 close } - assert {[r llen connected] > 1} - assert {[r llen disconnected] > 1} + assert {[r hooks.event_count client-connected] > 1} + assert {[r hooks.event_count client-disconnected] > 1} + } + + test {Test module cron hook} { + after 100 + assert {[r hooks.event_count cron-loop] > 0} + set hz [r hooks.event_last cron-loop] + assert_equal $hz 10 + } + + test {Test module loaded / unloaded hooks} { + set othermodule [file normalize tests/modules/infotest.so] + r module load $othermodule + r module unload infotest + assert_equal [r hooks.event_last module-loaded] "infotest" + assert_equal [r hooks.event_last module-unloaded] "infotest" + } + + test {Test module aofrw hook} { + r debug populate 1000 foo 10000 ;# 10mb worth of data + r config set rdbcompression no ;# rdb progress is only checked once in 2mb + r BGREWRITEAOF + waitForBgrewriteaof r + assert_equal [string match {*module-event-persistence-aof-start*} [exec tail -20 < [srv 0 stdout]]] 1 + assert_equal [string match {*module-event-persistence-end*} [exec tail -20 < [srv 0 stdout]]] 1 + } + + test {Test module aof load and rdb/aof progress hooks} { + # create some aof tail (progress is checked only once in 1000 commands) + for {set j 0} {$j < 4000} {incr j} { + r set "bar$j" x + } + # set some configs that will cause many loading progress events during aof loading + r config set key-load-delay 1 + r config set dynamic-hz no + r config set hz 500 + r DEBUG LOADAOF + assert_equal [r hooks.event_last loading-aof-start] 0 + assert_equal [r hooks.event_last loading-end] 0 + assert {[r hooks.event_count loading-rdb-start] == 0} + assert {[r hooks.event_count loading-progress-rdb] >= 2} ;# comes from the preamble section + assert {[r hooks.event_count loading-progress-aof] >= 2} + } + # undo configs before next test + r config set dynamic-hz yes + r config set key-load-delay 0 + + test {Test module rdb save hook} { + # debug reload does: save, flush, load: + assert {[r hooks.event_count persistence-syncrdb-start] == 0} + assert {[r hooks.event_count loading-rdb-start] == 0} + r debug reload + assert {[r hooks.event_count persistence-syncrdb-start] == 1} + assert {[r hooks.event_count loading-rdb-start] == 1} } test {Test flushdb hooks} { - r flushall ;# Note: only the "end" RPUSH will survive - r select 1 r flushdb - r select 2 - r flushdb - r select 9 - assert {[r llen flush-start] == 2} - assert {[r llen flush-end] == 3} - assert {[r lrange flush-start 0 -1] eq {1 2}} - assert {[r lrange flush-end 0 -1] eq {-1 1 2}} + assert_equal [r hooks.event_last flush-start] 9 + assert_equal [r hooks.event_last flush-end] 9 + r flushall + assert_equal [r hooks.event_last flush-start] -1 + assert_equal [r hooks.event_last flush-end] -1 } + + # replication related tests + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + start_server {} { + r module load $testmodule + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + $replica replicaof $master_host $master_port + + wait_for_condition 50 100 { + [string match {*master_link_status:up*} [r info replication]] + } else { + fail "Can't turn the instance into a replica" + } + + test {Test master link up hook} { + assert_equal [r hooks.event_count masterlink-up] 1 + assert_equal [r hooks.event_count masterlink-down] 0 + } + + test {Test role-replica hook} { + assert_equal [r hooks.event_count role-replica] 1 + assert_equal [r hooks.event_count role-master] 0 + assert_equal [r hooks.event_last role-replica] [s 0 master_host] + } + + test {Test replica-online hook} { + assert_equal [r -1 hooks.event_count replica-online] 1 + assert_equal [r -1 hooks.event_count replica-offline] 0 + } + + test {Test master link down hook} { + r client kill type master + assert_equal [r hooks.event_count masterlink-down] 1 + } + + $replica replicaof no one + + test {Test role-master hook} { + assert_equal [r hooks.event_count role-replica] 1 + assert_equal [r hooks.event_count role-master] 1 + assert_equal [r hooks.event_last role-master] {} + } + + test {Test replica-offline hook} { + assert_equal [r -1 hooks.event_count replica-online] 1 + assert_equal [r -1 hooks.event_count replica-offline] 1 + } + # get the replica stdout, to be used by the next test + set replica_stdout [srv 0 stdout] + } + + + # look into the log file of the server that just exited + test {Test shutdown hook} { + assert_equal [string match {*module-event-shutdown*} [exec tail -5 < $replica_stdout]] 1 + } + } } From 5f2a2b974e2ff329d9156cf6328160fdb9727a9c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2019 10:11:58 +0100 Subject: [PATCH 231/320] Modules: block on keys functions layout and mechanism. --- src/module.c | 153 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 120 insertions(+), 33 deletions(-) diff --git a/src/module.c b/src/module.c index 971bf5c08..bdafc1590 100644 --- a/src/module.c +++ b/src/module.c @@ -245,6 +245,15 @@ typedef struct RedisModuleBlockedClient { client *reply_client; /* Fake client used to accumulate replies in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ + int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ + int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, + void *privdata); /* When blocking on keys, even if the + key is signaled as ready, maybe it + was modified afterward before the + client unblocks. So we always + need a callback that tells us if + the key is ready in order to serve + the next blocked client. */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -3989,6 +3998,68 @@ void unblockClientFromModule(client *c) { resetClient(c); } +/* Block a client in the context of a module: this function implements both + * RM_BlockClient() and RM_BlockClientOnKeys() depending on the fact the + * keys are passed or not. + * + * When not blocking for keys, the keys, numkeys, is_key_ready callback + * and privdata parameters are not needed. The privdata in that case must + * be NULL, since later is RM_UnblockClient() that will provide some private + * data that the reply callback will receive. + * + * Instead when blocking for keys, normally RM_UnblockClient() will not be + * called (because the client will unblock when the key is modified), so + * 'privdata' should be provided in that case, so that once the client is + * unlocked and the reply callback is called, it will receive its associated + * private data. + * + * Even when blocking on keys, RM_UnblockClient() can be called however, but + * in that case the privdata argument is disregarded, because we pass the + * reply callback the privdata that is set here while blocking. + */ +RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *key, void *privdata), void *privdata) { + client *c = ctx->client; + int islua = c->flags & CLIENT_LUA; + int ismulti = c->flags & CLIENT_MULTI; + + c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + ctx->module->blocked_clients++; + + /* We need to handle the invalid operation of calling modules blocking + * commands from Lua or MULTI. We actually create an already aborted + * (client set to NULL) blocked client handle, and actually reply with + * an error. */ + mstime_t timeout = timeout_ms ? (mstime()+timeout_ms) : 0; + bc->client = (islua || ismulti) ? NULL : c; + bc->module = ctx->module; + bc->reply_callback = reply_callback; + bc->timeout_callback = timeout_callback; + bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ + bc->free_privdata = free_privdata; + bc->privdata = privdata; + bc->reply_client = createClient(NULL); + bc->reply_client->flags |= CLIENT_MODULE; + bc->dbid = c->db->id; + bc->is_key_ready = is_key_ready; + bc->blocked_on_keys = keys != NULL; + c->bpop.timeout = timeout; + + if (islua || ismulti) { + c->bpop.module_blocked_handle = NULL; + addReplyError(c, islua ? + "Blocking module command called from Lua script" : + "Blocking module command called from transaction"); + } else { + if (keys) { + blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,NULL,NULL); + } else { + blockClient(c,BLOCKED_MODULE); + } + } + return bc; +} + /* Block a client in the context of a blocking command, returning an handle * which will be used, later, in order to unblock the client with a call to * RedisModule_UnblockClient(). The arguments specify callback functions @@ -4006,39 +4077,55 @@ void unblockClientFromModule(client *c) { * by RedisModule_UnblockClient() call. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { - client *c = ctx->client; - int islua = c->flags & CLIENT_LUA; - int ismulti = c->flags & CLIENT_MULTI; + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL,NULL); +} - c->bpop.module_blocked_handle = zmalloc(sizeof(RedisModuleBlockedClient)); - RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; - ctx->module->blocked_clients++; - - /* We need to handle the invalid operation of calling modules blocking - * commands from Lua or MULTI. We actually create an already aborted - * (client set to NULL) blocked client handle, and actually reply with - * an error. */ - bc->client = (islua || ismulti) ? NULL : c; - bc->module = ctx->module; - bc->reply_callback = reply_callback; - bc->timeout_callback = timeout_callback; - bc->disconnect_callback = NULL; /* Set by RM_SetDisconnectCallback() */ - bc->free_privdata = free_privdata; - bc->privdata = NULL; - bc->reply_client = createClient(NULL); - bc->reply_client->flags |= CLIENT_MODULE; - bc->dbid = c->db->id; - c->bpop.timeout = timeout_ms ? (mstime()+timeout_ms) : 0; - - if (islua || ismulti) { - c->bpop.module_blocked_handle = NULL; - addReplyError(c, islua ? - "Blocking module command called from Lua script" : - "Blocking module command called from transaction"); - } else { - blockClient(c,BLOCKED_MODULE); - } - return bc; +/* This call is similar to RedisModule_BlockClient(), however in this case we + * don't just block the client, but also ask Redis to unblock it automatically + * once certain keys become "ready", that is, contain more data. + * + * Basically this is similar to what a typical Redis command usually does, + * like BLPOP or ZPOPMAX: the client blocks if it cannot be served ASAP, + * and later when the key receives new data (a list push for instance), the + * client is unblocked and served. + * + * However in the case of this module API, when the client is unblocked? + * + * 1. If you block ok a key of a type that has blocking operations associated, + * like a list, a sorted set, a stream, and so forth, the client may be + * unblocked once the relevant key is targeted by an operation that normally + * unblocks the native blocking operations for that type. So if we block + * on a list key, an RPUSH command may unblock our client and so forth. + * 2. If you are implementing your native data type, or if you want to add new + * unblocking conditions in addition to "1", you can call the modules API + * RedisModule_SignalKeyAsReady(). + * + * Anyway we can't be sure if the client should be unblocked just because the + * key is signaled as ready: for instance a successive operation may change the + * key, or a client in queue before this one can be served, modifying the key + * as well and making it empty again. So when blocking for keys, we need to + * register a callback called is_key_ready. This callback gets called with + * a context, selected with the right database, and the key name: if it + * returns 1, then we proceed calling the reply callback, otherwise if the + * is_key_ready callback returns 0 the client is not unblocked, since the + * key is yet not ready. + * + * Thanks to this system we can setup complex blocking scenarios, like + * unblocking a client only if a list contains at least 5 items or other + * more fancy logics. + * + * Note that another difference with RedisModule_BlockClient(), is that here + * we pass the private data directly when blocking the client: such private + * data will be later provided both to the is_key_ready callback, and to the + * reply callback. Normally in RedisModule_BlockClient() the private data + * to reply to the client is passed when calling RedisModule_UnblockClient() + * but here the unblocking is performed by Redis itself, so we need to have + * some private data before hand. The private data is used to store any + * information about the specific unblocking operation that you are + * implementing. Such information will be freed using the free_privdata + * callback provided by the user. */ +RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, void *privdata), void *privdata) { + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,is_key_ready,privdata); } /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger @@ -4054,7 +4141,7 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * Note: this function can be called from threads spawned by the module. */ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) { pthread_mutex_lock(&moduleUnblockedClientsMutex); - bc->privdata = privdata; + if (!bc->blocked_on_keys) bc->privdata = privdata; listAddNodeTail(moduleUnblockedClients,bc); if (write(server.module_blocked_pipe[1],"A",1) != 1) { /* Ignore the error, this is best-effort. */ From fb56b8fd0745b425af8578f025e13f7f9525933e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2019 10:20:28 +0100 Subject: [PATCH 232/320] Modules: block on keys: export APIs. --- src/module.c | 11 +++++++++++ src/redismodule.h | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/src/module.c b/src/module.c index bdafc1590..113375e75 100644 --- a/src/module.c +++ b/src/module.c @@ -4128,6 +4128,15 @@ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleC return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,is_key_ready,privdata); } +/* This function is used in order to potentially unblock a client blocked + * on keys with RedisModule_BlockClientOnKeys(). When this function is called, + * all the clients blocked for this key will get their is_key_ready callback, + * and if the callback returns non-zero the client will be unblocked and the + * reply callback will be called in order to reply to the client. */ +void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { + signalKeyAsReady(ctx->client->db, key); +} + /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger * the reply callbacks to be called in order to reply to the client. * The 'privdata' argument will be accessible by the reply callback, so @@ -6634,4 +6643,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldULongLong); REGISTER_API(GetClientInfoById); REGISTER_API(SubscribeToServerEvent); + REGISTER_API(BlockClientOnKeys); + REGISTER_API(SignalKeyAsReady); } diff --git a/src/redismodule.h b/src/redismodule.h index 4b63a227c..5b4c31b19 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -492,6 +492,8 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); +RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, void *privdata), void *privdata); +void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -688,6 +690,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldULongLong); REDISMODULE_GET_API(GetClientInfoById); REDISMODULE_GET_API(SubscribeToServerEvent); + REDISMODULE_GET_API(BlockClientOnKeys); + REDISMODULE_GET_API(SignalKeyAsReady); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); From c4becaebaacbd219577eb2e53756af2e9825f34d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2019 10:57:44 +0100 Subject: [PATCH 233/320] Modules: block on keys: implement the internals. --- src/blocked.c | 30 ++++++++++++++++++++++++++++ src/module.c | 54 +++++++++++++++++++++++++++++++++++++++++++-------- src/server.h | 2 ++ 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 867f03de6..2b91c1b44 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -430,6 +430,32 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { } } +/* Helper function for handleClientsBlockedOnKeys(). This function is called + * in order to check if we can serve clients blocked by modules using + * RM_BlockClientOnKeys(), when the corresponding key was signaled as ready: + * our goal here is to call the is_key_ready() callback to see if the key + * is really able to serve the client, and in that case, unblock it. */ +void serveClientsBlockedOnKeyByModule(readyList *rl) { + dictEntry *de; + + /* We serve clients in the same order they blocked for + * this key, from the first blocked to the last. */ + de = dictFind(rl->db->blocking_keys,rl->key); + if (de) { + list *clients = dictGetVal(de); + int numclients = listLength(clients); + + while(numclients--) { + listNode *clientnode = listFirst(clients); + client *receiver = clientnode->value; + + if (!moduleIsKeyReady(receiver, rl->key)) continue; + + moduleUnblockClient(receiver); + } + } +} + /* This function should be called by Redis every time a single command, * a MULTI/EXEC block, or a Lua script, terminated its execution after * being called by a client. It handles serving clients blocked in @@ -480,6 +506,10 @@ void handleClientsBlockedOnKeys(void) { serveClientsBlockedOnSortedSetKey(o,rl); else if (o->type == OBJ_STREAM) serveClientsBlockedOnStreamKey(o,rl); + /* We want to serve clients blocked on module keys + * regardless of the object type: we don't know what the + * module is trying to accomplish right now. */ + serveClientsBlockedOnKeyByModule(rl); } /* Free this item. */ diff --git a/src/module.c b/src/module.c index 113375e75..85bc9e808 100644 --- a/src/module.c +++ b/src/module.c @@ -4060,6 +4060,24 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF return bc; } +/* This function is called from module.c in order to check if a module + * blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true) + * can really be unblocked since the key is ready. */ +int moduleIsKeyReady(client *c, robj *key) { + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.module = bc->module; + ctx.client = bc->client; + ctx.blocked_privdata = bc->privdata; /* In case the callback uses the + API to get the pointer to the + privdata, even if we provide it + as argument. */ + selectDb(ctx.client, bc->dbid); + int ready = bc->is_key_ready(&ctx, key, bc->privdata); + moduleFreeContext(&ctx); + return ready; +} + /* Block a client in the context of a blocking command, returning an handle * which will be used, later, in order to unblock the client with a call to * RedisModule_UnblockClient(). The arguments specify callback functions @@ -4137,6 +4155,25 @@ void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { signalKeyAsReady(ctx->client->db, key); } +/* Implements RM_UnblockClient() and moduleUnblockClient(). */ +int moduleUnblockClientByHandle(RedisModuleBlockedClient *bc, void *privdata) { + pthread_mutex_lock(&moduleUnblockedClientsMutex); + if (!bc->blocked_on_keys) bc->privdata = privdata; + listAddNodeTail(moduleUnblockedClients,bc); + if (write(server.module_blocked_pipe[1],"A",1) != 1) { + /* Ignore the error, this is best-effort. */ + } + pthread_mutex_unlock(&moduleUnblockedClientsMutex); + return REDISMODULE_OK; +} + +/* This API is used by the Redis core to unblock a client that was blocked + * by a module. */ +void moduleUnblockClient(client *c) { + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + moduleUnblockClientByHandle(bc,NULL); +} + /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger * the reply callbacks to be called in order to reply to the client. * The 'privdata' argument will be accessible by the reply callback, so @@ -4147,15 +4184,16 @@ void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { * needs to be passed to the client, included but not limited some slow * to compute reply or some reply obtained via networking. * - * Note: this function can be called from threads spawned by the module. */ + * Note: this function can be called from threads spawned by the module. + * + * Note: when we unblock a client that is blocked for keys using + * the API RedisModule_BlockClientOnKeys(), the privdata argument here is + * not used, and the reply callback is called with the privdata pointer that + * was passed when blocking the client. Also note if you unblock clients + * blocked on keys in this way, the reply callback should be ready to handle + * the fact the key may not be ready at all. */ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) { - pthread_mutex_lock(&moduleUnblockedClientsMutex); - if (!bc->blocked_on_keys) bc->privdata = privdata; - listAddNodeTail(moduleUnblockedClients,bc); - if (write(server.module_blocked_pipe[1],"A",1) != 1) { - /* Ignore the error, this is best-effort. */ - } - pthread_mutex_unlock(&moduleUnblockedClientsMutex); + moduleUnblockClientByHandle(bc,privdata); return REDISMODULE_OK; } diff --git a/src/server.h b/src/server.h index 97672d727..3a2bb1c7b 100644 --- a/src/server.h +++ b/src/server.h @@ -1602,6 +1602,8 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); +int moduleIsKeyReady(client *c, robj *key); +void moduleUnblockClient(client *c); /* Utils */ long long ustime(void); From 5b3c740872ff909ec03f99c8f307c3f21946a611 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 10:30:54 +0100 Subject: [PATCH 234/320] Modules: block on keys: example on hellotype.c. --- src/modules/hellotype.c | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/modules/hellotype.c b/src/modules/hellotype.c index ba634c4a1..084408798 100644 --- a/src/modules/hellotype.c +++ b/src/modules/hellotype.c @@ -129,6 +129,7 @@ int HelloTypeInsert_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, /* Insert the new element. */ HelloTypeInsert(hto,value); + RedisModule_SignalKeyAsReady(ctx,argv[1]); RedisModule_ReplyWithLongLong(ctx,hto->len); RedisModule_ReplicateVerbatim(ctx); @@ -190,6 +191,69 @@ int HelloTypeLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int return REDISMODULE_OK; } +/* ====================== Example of a blocking command ==================== */ + +/* Is_key_ready callback for blocking command HELLOTYPE.BRANGE */ +int HelloBlock_IsKeyReady(RedisModuleCtx *ctx, RedisModuleString *keyname, void *privdata) { + REDISMODULE_NOT_USED(privdata); + + RedisModule_AutoMemory(ctx); /* Use automatic memory management. */ + RedisModuleKey *key = RedisModule_OpenKey(ctx,keyname,REDISMODULE_READ); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_MODULE || + RedisModule_ModuleTypeGetType(key) != HelloType) + { + return 0; + } else { + return 1; + } +} + +/* Reply callback for blocking command HELLOTYPE.BRANGE */ +int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); +} + +/* Timeout callback for blocking command HELLOTYPE.BRANGE */ +int HelloBlock_Timeout(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx,"Request timedout"); +} + +/* Private data freeing callback for HELLOTYPE.BRANGE command. */ +void HelloBlock_FreeData(RedisModuleCtx *ctx, void *privdata) { + REDISMODULE_NOT_USED(ctx); + RedisModule_Free(privdata); +} + +/* HELLOTYPE.BRANGE key first count timeout -- This is a blocking verison of + * the RANGE operation, in order to show how to use the API + * RedisModule_BlockClientOnKeys(). */ +int HelloTypeBRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 5) return RedisModule_WrongArity(ctx); + RedisModule_AutoMemory(ctx); /* Use automatic memory management. */ + RedisModuleKey *key = RedisModule_OpenKey(ctx,argv[1], + REDISMODULE_READ|REDISMODULE_WRITE); + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_EMPTY && + RedisModule_ModuleTypeGetType(key) != HelloType) + { + return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE); + } + + long long timeout; + if (RedisModule_StringToLongLong(argv[4],&timeout) != REDISMODULE_OK) { + return RedisModule_ReplyWithError(ctx, + "ERR invalid timeout parameter"); + } + + void *privdata = RedisModule_Alloc(100); + RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,HelloBlock_IsKeyReady,privdata); + return REDISMODULE_OK; +} /* ========================== "hellotype" type methods ======================= */ @@ -282,5 +346,9 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) HelloTypeLen_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"hellotype.brange", + HelloTypeBRange_RedisCommand,"readonly",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } From c9ad11d79707d8e2f82efa45fe4973136685b2bc Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 10:32:59 +0100 Subject: [PATCH 235/320] Modules: remove spurious call from moduleHandleBlockedClients(). Now we handle propagation when we free the context. --- src/module.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/module.c b/src/module.c index 85bc9e808..a11d3a306 100644 --- a/src/module.c +++ b/src/module.c @@ -4262,7 +4262,6 @@ void moduleHandleBlockedClients(void) { ctx.client = bc->client; ctx.blocked_client = bc; bc->reply_callback(&ctx,(void**)c->argv,c->argc); - moduleHandlePropagationAfterCommandCallback(&ctx); moduleFreeContext(&ctx); } From 4af1df6b9f4c4a9afbf745f8b684812eee2c64e4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 11:35:05 +0100 Subject: [PATCH 236/320] Modules: block on keys: use a better interface. Using the is_key_ready() callback plus the reply callback later, creates different issues AFAIK: 1. More complex API. 2. We need to call the reply callback() ASAP if the is_key_ready() interface returned success, however the internals do not work in that way, so when the reply callback is called the setup could be different. To fix that, there is to break the current design that handles the unblocked clients asyncrhonously, and run the list ASAP. --- src/blocked.c | 3 +- src/module.c | 74 +++++++++++++++++++++++++---------------- src/modules/hellotype.c | 26 +++++++-------- src/redismodule.h | 4 ++- src/server.h | 3 +- 5 files changed, 65 insertions(+), 45 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 2b91c1b44..fb58f850b 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -174,6 +174,7 @@ void unblockClient(client *c) { } else if (c->btype == BLOCKED_WAIT) { unblockClientWaitingReplicas(c); } else if (c->btype == BLOCKED_MODULE) { + if (moduleClientIsBlockedOnKeys(c)) unblockClientWaitingData(c); unblockClientFromModule(c); } else { serverPanic("Unknown btype in unblockClient()."); @@ -449,7 +450,7 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { listNode *clientnode = listFirst(clients); client *receiver = clientnode->value; - if (!moduleIsKeyReady(receiver, rl->key)) continue; + if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; moduleUnblockClient(receiver); } diff --git a/src/module.c b/src/module.c index a11d3a306..8837ae017 100644 --- a/src/module.c +++ b/src/module.c @@ -140,6 +140,9 @@ struct RedisModuleCtx { void **postponed_arrays; /* To set with RM_ReplySetArrayLength(). */ int postponed_arrays_count; /* Number of entries in postponed_arrays. */ void *blocked_privdata; /* Privdata set when unblocking a client. */ + RedisModuleString *blocked_ready_key; /* Key ready when the reply callback + gets called for clients blocked + on keys. */ /* Used if there is the REDISMODULE_CTX_KEYS_POS_REQUEST flag set. */ int *keys_pos; @@ -153,7 +156,7 @@ struct RedisModuleCtx { }; typedef struct RedisModuleCtx RedisModuleCtx; -#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, 0, NULL, {0}} +#define REDISMODULE_CTX_INIT {(void*)(unsigned long)&RM_GetApi, NULL, NULL, NULL, NULL, 0, 0, 0, NULL, 0, NULL, NULL, NULL, 0, NULL, {0}} #define REDISMODULE_CTX_MULTI_EMITTED (1<<0) #define REDISMODULE_CTX_AUTO_MEMORY (1<<1) #define REDISMODULE_CTX_KEYS_POS_REQUEST (1<<2) @@ -246,14 +249,6 @@ typedef struct RedisModuleBlockedClient { in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ - int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, - void *privdata); /* When blocking on keys, even if the - key is signaled as ready, maybe it - was modified afterward before the - client unblocks. So we always - need a callback that tells us if - the key is ready in order to serve - the next blocked client. */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -4002,10 +3997,10 @@ void unblockClientFromModule(client *c) { * RM_BlockClient() and RM_BlockClientOnKeys() depending on the fact the * keys are passed or not. * - * When not blocking for keys, the keys, numkeys, is_key_ready callback - * and privdata parameters are not needed. The privdata in that case must - * be NULL, since later is RM_UnblockClient() that will provide some private - * data that the reply callback will receive. + * When not blocking for keys, the keys, numkeys, and privdata parameters are + * not needed. The privdata in that case must be NULL, since later is + * RM_UnblockClient() that will provide some private data that the reply + * callback will receive. * * Instead when blocking for keys, normally RM_UnblockClient() will not be * called (because the client will unblock when the key is modified), so @@ -4017,7 +4012,7 @@ void unblockClientFromModule(client *c) { * in that case the privdata argument is disregarded, because we pass the * reply callback the privdata that is set here while blocking. */ -RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *key, void *privdata), void *privdata) { +RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { client *c = ctx->client; int islua = c->flags & CLIENT_LUA; int ismulti = c->flags & CLIENT_MULTI; @@ -4041,7 +4036,6 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF bc->reply_client = createClient(NULL); bc->reply_client->flags |= CLIENT_MODULE; bc->dbid = c->db->id; - bc->is_key_ready = is_key_ready; bc->blocked_on_keys = keys != NULL; c->bpop.timeout = timeout; @@ -4062,20 +4056,24 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF /* This function is called from module.c in order to check if a module * blocked for BLOCKED_MODULE and subtype 'on keys' (bc->blocked_on_keys true) - * can really be unblocked since the key is ready. */ -int moduleIsKeyReady(client *c, robj *key) { + * can really be unblocked, since the module was able to serve the client. + * If the callback returns REDISMODULE_OK, then the client can be unblocked, + * otherwise the client remains blocked and we'll retry again when one of + * the keys it blocked for becomes "ready" again. */ +int moduleTryServeClientBlockedOnKey(client *c, robj *key) { + int served = 0; RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; + ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; + ctx.blocked_ready_key = key; + ctx.blocked_privdata = bc->privdata; ctx.module = bc->module; ctx.client = bc->client; - ctx.blocked_privdata = bc->privdata; /* In case the callback uses the - API to get the pointer to the - privdata, even if we provide it - as argument. */ - selectDb(ctx.client, bc->dbid); - int ready = bc->is_key_ready(&ctx, key, bc->privdata); + ctx.blocked_client = bc; + if (bc->reply_callback(&ctx,(void**)c->argv,c->argc) == REDISMODULE_OK) + served = 1; moduleFreeContext(&ctx); - return ready; + return served; } /* Block a client in the context of a blocking command, returning an handle @@ -4095,7 +4093,7 @@ int moduleIsKeyReady(client *c, robj *key) { * by RedisModule_UnblockClient() call. */ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms) { - return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL,NULL); + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, NULL,0,NULL); } /* This call is similar to RedisModule_BlockClient(), however in this case we @@ -4142,8 +4140,8 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * information about the specific unblocking operation that you are * implementing. Such information will be freed using the free_privdata * callback provided by the user. */ -RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, void *privdata), void *privdata) { - return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,is_key_ready,privdata); +RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { + return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); } /* This function is used in order to potentially unblock a client blocked @@ -4174,6 +4172,13 @@ void moduleUnblockClient(client *c) { moduleUnblockClientByHandle(bc,NULL); } +/* Return true if the client 'c' was blocked by a module using + * RM_BlockClientOnKeys(). */ +int moduleClientIsBlockedOnKeys(client *c) { + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + return bc->blocked_on_keys; +} + /* Unblock a client blocked by `RedisModule_BlockedClient`. This will trigger * the reply callbacks to be called in order to reply to the client. * The 'privdata' argument will be accessible by the reply callback, so @@ -4253,11 +4258,15 @@ void moduleHandleBlockedClients(void) { * touch the shared list. */ /* Call the reply callback if the client is valid and we have - * any callback. */ - if (c && bc->reply_callback) { + * any callback. However the callback is not called if the client + * was blocked on keys (RM_BlockClientOnKeys()), because we already + * called such callback in moduleTryServeClientBlockedOnKey() when + * the key was signaled as ready. */ + if (c && !bc->blocked_on_keys && bc->reply_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_privdata = bc->privdata; + ctx.blocked_ready_key = NULL; ctx.module = bc->module; ctx.client = bc->client; ctx.blocked_client = bc; @@ -4349,6 +4358,12 @@ void *RM_GetBlockedClientPrivateData(RedisModuleCtx *ctx) { return ctx->blocked_privdata; } +/* Get the key that is ready when the reply callback is called in the context + * of a client blocked by RedisModule_BlockClientOnKeys(). */ +RedisModuleString *RM_GetBlockedClientReadyKey(RedisModuleCtx *ctx) { + return ctx->blocked_ready_key; +} + /* Get the blocked client associated with a given context. * This is useful in the reply and timeout callbacks of blocked clients, * before sometimes the module has the blocked client handle references @@ -6682,4 +6697,5 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(SubscribeToServerEvent); REGISTER_API(BlockClientOnKeys); REGISTER_API(SignalKeyAsReady); + REGISTER_API(GetBlockedClientReadyKey); } diff --git a/src/modules/hellotype.c b/src/modules/hellotype.c index 084408798..dafbadbe5 100644 --- a/src/modules/hellotype.c +++ b/src/modules/hellotype.c @@ -193,26 +193,26 @@ int HelloTypeLen_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int /* ====================== Example of a blocking command ==================== */ -/* Is_key_ready callback for blocking command HELLOTYPE.BRANGE */ -int HelloBlock_IsKeyReady(RedisModuleCtx *ctx, RedisModuleString *keyname, void *privdata) { - REDISMODULE_NOT_USED(privdata); +/* Reply callback for blocking command HELLOTYPE.BRANGE, this will get + * called when the key we blocked for is ready: we need to check if we + * can really serve the client, and reply OK or ERR accordingly. */ +int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); - RedisModule_AutoMemory(ctx); /* Use automatic memory management. */ + RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); RedisModuleKey *key = RedisModule_OpenKey(ctx,keyname,REDISMODULE_READ); int type = RedisModule_KeyType(key); if (type != REDISMODULE_KEYTYPE_MODULE || RedisModule_ModuleTypeGetType(key) != HelloType) { - return 0; - } else { - return 1; + RedisModule_CloseKey(key); + return REDISMODULE_ERR; } -} -/* Reply callback for blocking command HELLOTYPE.BRANGE */ -int HelloBlock_Reply(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - REDISMODULE_NOT_USED(argv); - REDISMODULE_NOT_USED(argc); + /* In case the key is able to serve our blocked client, let's directly + * use our original command implementation to make this example simpler. */ + RedisModule_CloseKey(key); return HelloTypeRange_RedisCommand(ctx,argv,argc-1); } @@ -251,7 +251,7 @@ int HelloTypeBRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, } void *privdata = RedisModule_Alloc(100); - RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,HelloBlock_IsKeyReady,privdata); + RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,privdata); return REDISMODULE_OK; } diff --git a/src/redismodule.h b/src/redismodule.h index 5b4c31b19..1b284770b 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -492,8 +492,9 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); -RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, int (*is_key_ready)(RedisModuleCtx*, RedisModuleString *keyname, void *privdata), void *privdata); +RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientReadyKey)(RedisModuleCtx *ctx); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -692,6 +693,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(SubscribeToServerEvent); REDISMODULE_GET_API(BlockClientOnKeys); REDISMODULE_GET_API(SignalKeyAsReady); + REDISMODULE_GET_API(GetBlockedClientReadyKey); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); diff --git a/src/server.h b/src/server.h index 3a2bb1c7b..f724f7d64 100644 --- a/src/server.h +++ b/src/server.h @@ -1602,8 +1602,9 @@ ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); -int moduleIsKeyReady(client *c, robj *key); +int moduleTryServeClientBlockedOnKey(client *c, robj *key); void moduleUnblockClient(client *c); +int moduleClientIsBlockedOnKeys(client *c); /* Utils */ long long ustime(void); From 08df2470a9fc159a4967ee559e3e871f4dd5867d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 11:43:45 +0100 Subject: [PATCH 237/320] Modules: block on keys: fix the top comments. --- src/module.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/module.c b/src/module.c index 8837ae017..353b6f426 100644 --- a/src/module.c +++ b/src/module.c @@ -4119,36 +4119,41 @@ RedisModuleBlockedClient *RM_BlockClient(RedisModuleCtx *ctx, RedisModuleCmdFunc * Anyway we can't be sure if the client should be unblocked just because the * key is signaled as ready: for instance a successive operation may change the * key, or a client in queue before this one can be served, modifying the key - * as well and making it empty again. So when blocking for keys, we need to - * register a callback called is_key_ready. This callback gets called with - * a context, selected with the right database, and the key name: if it - * returns 1, then we proceed calling the reply callback, otherwise if the - * is_key_ready callback returns 0 the client is not unblocked, since the - * key is yet not ready. + * as well and making it empty again. So when a client is blocked with + * RedisModule_BlockClientOnKeys() the reply callback is not called after + * RM_UnblockCLient() is called, but every time a key is signaled as ready: + * if the reply callback can serve the client, it returns REDISMODULE_OK + * and the client is unblocked, otherwise it will return REDISMODULE_ERR + * and we'll try again later. + * + * The reply callback can access the key that was signaled as ready by + * calling the API RedisModule_GetBlockedClientReadyKey(), that returns + * just the string name of the key as a RedisModuleString object. * * Thanks to this system we can setup complex blocking scenarios, like * unblocking a client only if a list contains at least 5 items or other * more fancy logics. * * Note that another difference with RedisModule_BlockClient(), is that here - * we pass the private data directly when blocking the client: such private - * data will be later provided both to the is_key_ready callback, and to the - * reply callback. Normally in RedisModule_BlockClient() the private data - * to reply to the client is passed when calling RedisModule_UnblockClient() - * but here the unblocking is performed by Redis itself, so we need to have - * some private data before hand. The private data is used to store any - * information about the specific unblocking operation that you are - * implementing. Such information will be freed using the free_privdata - * callback provided by the user. */ + * we pass the private data directly when blocking the client: it will + * be accessible later in the reply callback. Normally when blocking with + * RedisModule_BlockClient() the private data to reply to the client is + * passed when calling RedisModule_UnblockClient() but here the unblocking + * is performed by Redis itself, so we need to have some private data before + * hand. The private data is used to store any information about the specific + * unblocking operation that you are implementing. Such information will be + * freed using the free_privdata callback provided by the user. + * + * However the reply callback will be able to access the argument vector of + * the command, so the private data is often not needed. */ RedisModuleBlockedClient *RM_BlockClientOnKeys(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata) { return moduleBlockClient(ctx,reply_callback,timeout_callback,free_privdata,timeout_ms, keys,numkeys,privdata); } /* This function is used in order to potentially unblock a client blocked * on keys with RedisModule_BlockClientOnKeys(). When this function is called, - * all the clients blocked for this key will get their is_key_ready callback, - * and if the callback returns non-zero the client will be unblocked and the - * reply callback will be called in order to reply to the client. */ + * all the clients blocked for this key will get their reply callback called, + * and if the callback returns REDISMODULE_OK the client will be unblocked. */ void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { signalKeyAsReady(ctx->client->db, key); } From e6eaf0fce6aa8bbf17bd48e10b3c74fef62e9a05 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 12:23:55 +0100 Subject: [PATCH 238/320] Modules: block on keys: fix bugs in processing order. --- src/blocked.c | 16 ++++++++++++++++ src/module.c | 7 +++++++ 2 files changed, 23 insertions(+) diff --git a/src/blocked.c b/src/blocked.c index fb58f850b..3110c00fc 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -450,6 +450,22 @@ void serveClientsBlockedOnKeyByModule(readyList *rl) { listNode *clientnode = listFirst(clients); client *receiver = clientnode->value; + /* Put at the tail, so that at the next call + * we'll not run into it again: clients here may not be + * ready to be served, so they'll remain in the list + * sometimes. We want also be able to skip clients that are + * not blocked for the MODULE type safely. */ + listDelNode(clients,clientnode); + listAddNodeTail(clients,receiver); + + if (receiver->btype != BLOCKED_MODULE) continue; + + /* Note that if *this* client cannot be served by this key, + * it does not mean that another client that is next into the + * list cannot be served as well: they may be blocked by + * different modules with different triggers to consider if a key + * is ready or not. This means we can't exit the loop but need + * to continue after the first failure. */ if (!moduleTryServeClientBlockedOnKey(receiver, rl->key)) continue; moduleUnblockClient(receiver); diff --git a/src/module.c b/src/module.c index 353b6f426..248a55e62 100644 --- a/src/module.c +++ b/src/module.c @@ -249,6 +249,7 @@ typedef struct RedisModuleBlockedClient { in thread safe contexts. */ int dbid; /* Database number selected by the original client. */ int blocked_on_keys; /* If blocked via RM_BlockClientOnKeys(). */ + int unblocked; /* Already on the moduleUnblocked list. */ } RedisModuleBlockedClient; static pthread_mutex_t moduleUnblockedClientsMutex = PTHREAD_MUTEX_INITIALIZER; @@ -4037,6 +4038,7 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF bc->reply_client->flags |= CLIENT_MODULE; bc->dbid = c->db->id; bc->blocked_on_keys = keys != NULL; + bc->unblocked = 0; c->bpop.timeout = timeout; if (islua || ismulti) { @@ -4063,6 +4065,10 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF int moduleTryServeClientBlockedOnKey(client *c, robj *key) { int served = 0; RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + /* Protect against re-processing: don't serve clients that are already + * in the unblocking list for any reason (including RM_UnblockClient() + * explicit call). */ + if (bc->unblocked) return REDISMODULE_ERR; RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.flags |= REDISMODULE_CTX_BLOCKED_REPLY; ctx.blocked_ready_key = key; @@ -4162,6 +4168,7 @@ void RM_SignalKeyAsReady(RedisModuleCtx *ctx, RedisModuleString *key) { int moduleUnblockClientByHandle(RedisModuleBlockedClient *bc, void *privdata) { pthread_mutex_lock(&moduleUnblockedClientsMutex); if (!bc->blocked_on_keys) bc->privdata = privdata; + bc->unblocked = 1; listAddNodeTail(moduleUnblockedClients,bc); if (write(server.module_blocked_pipe[1],"A",1) != 1) { /* Ignore the error, this is best-effort. */ From 27f27268460f52b285d136ad12be73053f58cfc6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 12:31:22 +0100 Subject: [PATCH 239/320] Modules: block ok keys: improve example. --- src/modules/hellotype.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/modules/hellotype.c b/src/modules/hellotype.c index dafbadbe5..4f2d1d730 100644 --- a/src/modules/hellotype.c +++ b/src/modules/hellotype.c @@ -244,12 +244,20 @@ int HelloTypeBRange_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, return RedisModule_ReplyWithError(ctx,REDISMODULE_ERRORMSG_WRONGTYPE); } + /* Parse the timeout before even trying to serve the client synchronously, + * so that we always fail ASAP on syntax errors. */ long long timeout; if (RedisModule_StringToLongLong(argv[4],&timeout) != REDISMODULE_OK) { return RedisModule_ReplyWithError(ctx, "ERR invalid timeout parameter"); } + /* Can we serve the reply synchronously? */ + if (type != REDISMODULE_KEYTYPE_EMPTY) { + return HelloTypeRange_RedisCommand(ctx,argv,argc-1); + } + + /* Otherwise let's block on the key. */ void *privdata = RedisModule_Alloc(100); RedisModule_BlockClientOnKeys(ctx,HelloBlock_Reply,HelloBlock_Timeout,HelloBlock_FreeData,timeout,argv+1,1,privdata); return REDISMODULE_OK; From 3b10c398f977a18de683bb1a842d270e8950adf5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 17:38:58 +0100 Subject: [PATCH 240/320] Modules: block on keys: finish implementing RM_UnblockClient(). --- src/module.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 248a55e62..a4384163a 100644 --- a/src/module.c +++ b/src/module.c @@ -4201,15 +4201,24 @@ int moduleClientIsBlockedOnKeys(client *c) { * needs to be passed to the client, included but not limited some slow * to compute reply or some reply obtained via networking. * - * Note: this function can be called from threads spawned by the module. + * Note 1: this function can be called from threads spawned by the module. * - * Note: when we unblock a client that is blocked for keys using + * Note 2: when we unblock a client that is blocked for keys using * the API RedisModule_BlockClientOnKeys(), the privdata argument here is * not used, and the reply callback is called with the privdata pointer that - * was passed when blocking the client. Also note if you unblock clients - * blocked on keys in this way, the reply callback should be ready to handle - * the fact the key may not be ready at all. */ + * was passed when blocking the client. + * + * Unblocking a client that was blocked for keys using this API will still + * require the client to get some reply, so the function will use the + * "timeout" handler in order to do so. */ int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) { + if (bc->blocked_on_keys) { + /* In theory the user should always pass the timeout handler as an + * argument, but better to be safe than sorry. */ + if (bc->timeout_callback == NULL) return REDISMODULE_ERR; + if (bc->unblocked) return REDISMODULE_OK; + if (bc->client) moduleBlockedClientTimedOut(bc->client); + } moduleUnblockClientByHandle(bc,privdata); return REDISMODULE_OK; } From 44a2ebe81e4c0c5cc2980230faa490ec8b295919 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 17:45:07 +0100 Subject: [PATCH 241/320] Modules: block on keys: fix stale comment. --- src/blocked.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 3110c00fc..14c2ff830 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -434,8 +434,9 @@ void serveClientsBlockedOnStreamKey(robj *o, readyList *rl) { /* Helper function for handleClientsBlockedOnKeys(). This function is called * in order to check if we can serve clients blocked by modules using * RM_BlockClientOnKeys(), when the corresponding key was signaled as ready: - * our goal here is to call the is_key_ready() callback to see if the key - * is really able to serve the client, and in that case, unblock it. */ + * our goal here is to call the RedisModuleBlockedClient reply() callback to + * see if the key is really able to serve the client, and in that case, + * unblock it. */ void serveClientsBlockedOnKeyByModule(readyList *rl) { dictEntry *de; From d44586c6aa9e7f0e78ac143220479d66bd47ec72 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2019 18:07:33 +0100 Subject: [PATCH 242/320] Modules: fix thread safe context creation crash. See #6525, this likely creates a NULL deference if the client was terminated by Redis between the creation of the blocked client and the creation of the thread safe context. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index f298ec760..f9f654b42 100644 --- a/src/module.c +++ b/src/module.c @@ -4463,7 +4463,7 @@ RedisModuleCtx *RM_GetThreadSafeContext(RedisModuleBlockedClient *bc) { ctx->client = createClient(NULL); if (bc) { selectDb(ctx->client,bc->dbid); - ctx->client->id = bc->client->id; + if (bc->client) ctx->client->id = bc->client->id; } return ctx; } From 2bbd305a33ef31f0ea34991055ff188f3d3cfbe9 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 3 Nov 2019 15:02:25 +0200 Subject: [PATCH 243/320] Add module api for looking into INFO fields - Add RM_GetServerInfo and friends - Add auto memory for new opaque struct - Add tests for new APIs other minor fixes: - add const in various char pointers - requested_section in modulesCollectInfo was actually not sds but char* - extract new string2d out of getDoubleFromObject for code reuse Add module API for --- src/module.c | 112 +++++++++++++++++++++++++++++- src/object.c | 10 +-- src/redismodule.h | 11 +++ src/server.c | 2 +- src/server.h | 4 +- src/util.c | 21 ++++++ src/util.h | 1 + tests/modules/infotest.c | 52 ++++++++++++++ tests/unit/moduleapi/infotest.tcl | 21 ++++++ 9 files changed, 220 insertions(+), 14 deletions(-) diff --git a/src/module.c b/src/module.c index f9f654b42..ce85c479d 100644 --- a/src/module.c +++ b/src/module.c @@ -41,7 +41,7 @@ typedef struct RedisModuleInfoCtx { struct RedisModule *module; - sds requested_section; + const char *requested_section; sds info; /* info string we collected so far */ int sections; /* number of sections we collected so far */ int in_section; /* indication if we're in an active section or not */ @@ -93,6 +93,7 @@ struct AutoMemEntry { #define REDISMODULE_AM_REPLY 2 #define REDISMODULE_AM_FREED 3 /* Explicitly freed by user already. */ #define REDISMODULE_AM_DICT 4 +#define REDISMODULE_AM_INFO 5 /* The pool allocator block. Redis Modules can allocate memory via this special * allocator that will automatically release it all once the callback returns. @@ -322,6 +323,10 @@ static struct RedisModuleForkInfo { void* done_handler_user_data; } moduleForkInfo = {0}; +typedef struct RedisModuleServerInfoData { + rax *rax; /* parsed info data. */ +} RedisModuleServerInfoData; + /* Flags for moduleCreateArgvFromUserFormat(). */ #define REDISMODULE_ARGV_REPLICATE (1<<0) #define REDISMODULE_ARGV_NO_AOF (1<<1) @@ -361,6 +366,7 @@ void moduleReplicateMultiIfNeeded(RedisModuleCtx *ctx); void RM_ZsetRangeStop(RedisModuleKey *kp); static void zsetKeyReset(RedisModuleKey *key); void RM_FreeDict(RedisModuleCtx *ctx, RedisModuleDict *d); +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); /* -------------------------------------------------------------------------- * Heap allocation raw functions @@ -946,6 +952,7 @@ void autoMemoryCollect(RedisModuleCtx *ctx) { case REDISMODULE_AM_REPLY: RM_FreeCallReply(ptr); break; case REDISMODULE_AM_KEY: RM_CloseKey(ptr); break; case REDISMODULE_AM_DICT: RM_FreeDict(NULL,ptr); break; + case REDISMODULE_AM_INFO: RM_FreeServerInfo(NULL,ptr); break; } } ctx->flags |= REDISMODULE_CTX_AUTO_MEMORY; @@ -5449,7 +5456,7 @@ int RM_RegisterInfoFunc(RedisModuleCtx *ctx, RedisModuleInfoFunc cb) { return REDISMODULE_OK; } -sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections) { +sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections) { dictIterator *di = dictGetIterator(modules); dictEntry *de; @@ -5469,6 +5476,102 @@ sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections return info; } +/* Get information about the server similar to the one that returns from the + * INFO command. This function takes an optional 'section' argument that may + * be NULL. The return value holds the output and can be used with + * RedisModule_ServerInfoGetField and alike to get the individual fields. + * When done, it needs to be freed with RedisModule_FreeServerInfo or with the + * automatic memory management mechanism if enabled. */ +RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *section) { + struct RedisModuleServerInfoData *d = zmalloc(sizeof(*d)); + d->rax = raxNew(); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_INFO,d); + sds info = genRedisInfoString(section); + int totlines, i; + sds *lines = sdssplitlen(info, sdslen(info), "\r\n", 2, &totlines); + for(i=0; irax,key,keylen,val,NULL); + } + sdsfree(info); + sdsfreesplitres(lines,totlines); + return d; +} + +/* Free data created with RM_GetServerInfo(). You need to pass the + * context pointer 'ctx' only if the dictionary was created using the + * context instead of passing NULL. */ +void RM_FreeServerInfo(RedisModuleCtx *ctx, RedisModuleServerInfoData *data) { + if (ctx != NULL) autoMemoryFreed(ctx,REDISMODULE_AM_INFO,data); + raxIterator ri; + raxStart(&ri,data->rax); + while(1) { + raxSeek(&ri,"^",NULL,0); + if (!raxNext(&ri)) break; + raxRemove(data->rax,(unsigned char*)ri.key,ri.key_len,NULL); + sdsfree(ri.data); + } + raxStop(&ri); + raxFree(data->rax); + zfree(data); +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). You + * need to pass the context pointer 'ctx' only if you want to use auto memory + * mechanism to release the returned string. Return value will be NULL if the + * field was not found. */ +RedisModuleString *RM_ServerInfoGetField(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field) { + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + RedisModuleString *o = createStringObject(val,sdslen(val)); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,o); + return o; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical, return value will be 0, and the + * optional out_err argument will be set to REDISMODULE_ERR. */ +long long RM_ServerInfoGetFieldNumerical(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err) { + UNUSED(ctx); + long long ll; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ll(val,sdslen(val),&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not a double, return value will be 0, and the + * optional out_err argument will be set to REDISMODULE_ERR. */ +double RM_ServerInfoGetFieldDouble(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err) { + UNUSED(ctx); + double dbl; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2d(val,sdslen(val),&dbl)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return dbl; +} + /* -------------------------------------------------------------------------- * Modules utility APIs * -------------------------------------------------------------------------- */ @@ -6756,6 +6859,11 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldDouble); REGISTER_API(InfoAddFieldLongLong); REGISTER_API(InfoAddFieldULongLong); + REGISTER_API(GetServerInfo); + REGISTER_API(FreeServerInfo); + REGISTER_API(ServerInfoGetField); + REGISTER_API(ServerInfoGetFieldNumerical); + REGISTER_API(ServerInfoGetFieldDouble); REGISTER_API(GetClientInfoById); REGISTER_API(SubscribeToServerEvent); REGISTER_API(BlockClientOnKeys); diff --git a/src/object.c b/src/object.c index 70022f897..7ce79bbbb 100644 --- a/src/object.c +++ b/src/object.c @@ -606,21 +606,13 @@ size_t stringObjectLen(robj *o) { int getDoubleFromObject(const robj *o, double *target) { double value; - char *eptr; if (o == NULL) { value = 0; } else { serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); if (sdsEncodedObject(o)) { - errno = 0; - value = strtod(o->ptr, &eptr); - if (sdslen(o->ptr) == 0 || - isspace(((const char*)o->ptr)[0]) || - (size_t)(eptr-(char*)o->ptr) != sdslen(o->ptr) || - (errno == ERANGE && - (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || - isnan(value)) + if (!string2d(o->ptr, sdslen(o->ptr), &value)) return C_ERR; } else if (o->encoding == OBJ_ENCODING_INT) { value = (long)o->ptr; diff --git a/src/redismodule.h b/src/redismodule.h index ea0d6a139..d923a1b8a 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -318,6 +318,7 @@ typedef struct RedisModuleDictIter RedisModuleDictIter; typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx; typedef struct RedisModuleCommandFilter RedisModuleCommandFilter; typedef struct RedisModuleInfoCtx RedisModuleInfoCtx; +typedef struct RedisModuleServerInfoData RedisModuleServerInfoData; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -502,6 +503,11 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldCString)(RedisModuleInfoCtx *ct int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx, char *field, double value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); +RedisModuleServerInfoData *REDISMODULE_API_FUNC(RedisModule_GetServerInfo)(RedisModuleCtx *ctx, const char *section); +void REDISMODULE_API_FUNC(RedisModule_FreeServerInfo)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetField)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field); +long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldNumerical)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err); +double REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldDouble)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); @@ -704,6 +710,11 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldDouble); REDISMODULE_GET_API(InfoAddFieldLongLong); REDISMODULE_GET_API(InfoAddFieldULongLong); + REDISMODULE_GET_API(GetServerInfo); + REDISMODULE_GET_API(FreeServerInfo); + REDISMODULE_GET_API(ServerInfoGetField); + REDISMODULE_GET_API(ServerInfoGetFieldNumerical); + REDISMODULE_GET_API(ServerInfoGetFieldDouble); REDISMODULE_GET_API(GetClientInfoById); REDISMODULE_GET_API(SubscribeToServerEvent); REDISMODULE_GET_API(BlockClientOnKeys); diff --git a/src/server.c b/src/server.c index 8f165113d..a11cb538d 100644 --- a/src/server.c +++ b/src/server.c @@ -3909,7 +3909,7 @@ void bytesToHuman(char *s, unsigned long long n) { /* Create the string returned by the INFO command. This is decoupled * by the INFO command itself as we need to report the same information * on memory corruption problems. */ -sds genRedisInfoString(char *section) { +sds genRedisInfoString(const char *section) { sds info = sdsempty(); time_t uptime = server.unixtime-server.stat_starttime; int j; diff --git a/src/server.h b/src/server.h index f724f7d64..9932bafa8 100644 --- a/src/server.h +++ b/src/server.h @@ -1600,7 +1600,7 @@ void ModuleForkDoneHandler(int exitcode, int bysignal); int TerminateModuleForkChild(int child_pid, int wait); ssize_t rdbSaveModulesAux(rio *rdb, int when); int moduleAllDatatypesHandleErrors(); -sds modulesCollectInfo(sds info, sds section, int for_crash_report, int sections); +sds modulesCollectInfo(sds info, const char *section, int for_crash_report, int sections); void moduleFireServerEvent(uint64_t eid, int subid, void *data); int moduleTryServeClientBlockedOnKey(client *c, robj *key); void moduleUnblockClient(client *c); @@ -2409,7 +2409,7 @@ void _serverPanic(const char *file, int line, const char *msg, ...); void bugReportStart(void); void serverLogObjectDebugInfo(const robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); -sds genRedisInfoString(char *section); +sds genRedisInfoString(const char *section); sds genModulesInfoString(sds info); void enableWatchdog(int period); void disableWatchdog(void); diff --git a/src/util.c b/src/util.c index 783bcf83b..6e9dc2117 100644 --- a/src/util.c +++ b/src/util.c @@ -468,6 +468,27 @@ int string2ld(const char *s, size_t slen, long double *dp) { return 1; } +/* Convert a string into a double. Returns 1 if the string could be parsed + * into a (non-overflowing) double, 0 otherwise. The value will be set to + * the parsed value when appropriate. + * + * Note that this function demands that the string strictly represents + * a double: no spaces or other characters before or after the string + * representing the number are accepted. */ +int string2d(const char *s, size_t slen, double *dp) { + errno = 0; + char *eptr; + *dp = strtod(s, &eptr); + if (slen == 0 || + isspace(((const char*)s)[0]) || + (size_t)(eptr-(char*)s) != slen || + (errno == ERANGE && + (*dp == HUGE_VAL || *dp == -HUGE_VAL || *dp == 0)) || + isnan(*dp)) + return 0; + return 1; +} + /* Convert a double to a string representation. Returns the number of bytes * required. The representation should always be parsable by strtod(3). * This function does not support human-friendly formatting like ld2string diff --git a/src/util.h b/src/util.h index b6c01aa59..ab1d71f6c 100644 --- a/src/util.h +++ b/src/util.h @@ -48,6 +48,7 @@ int ll2string(char *s, size_t len, long long value); int string2ll(const char *s, size_t slen, long long *value); int string2l(const char *s, size_t slen, long *value); int string2ld(const char *s, size_t slen, long double *dp); +int string2d(const char *s, size_t slen, double *dp); int d2string(char *buf, size_t len, double value); int ld2string(char *buf, size_t len, long double value, int humanfriendly); sds getAbsolutePath(char *filename); diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index d28410932..460ebb9c0 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -29,6 +29,51 @@ void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { } +int info_get(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, char field_type) +{ + if (argc != 3 && argc != 4) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + int err = REDISMODULE_OK; + const char *section, *field; + section = RedisModule_StringPtrLen(argv[1], NULL); + field = RedisModule_StringPtrLen(argv[2], NULL); + RedisModuleServerInfoData *info = RedisModule_GetServerInfo(ctx, section); + if (field_type=='i') { + long long ll = RedisModule_ServerInfoGetFieldNumerical(ctx, info, field, &err); + if (err==REDISMODULE_OK) + RedisModule_ReplyWithLongLong(ctx, ll); + } else if (field_type=='d') { + double d = RedisModule_ServerInfoGetFieldDouble(ctx, info, field, &err); + if (err==REDISMODULE_OK) + RedisModule_ReplyWithDouble(ctx, d); + } else { + RedisModuleString *str = RedisModule_ServerInfoGetField(ctx, info, field); + if (str) { + RedisModule_ReplyWithString(ctx, str); + RedisModule_FreeString(ctx, str); + } else + err=REDISMODULE_ERR; + } + if (err!=REDISMODULE_OK) + RedisModule_ReplyWithError(ctx, "not found"); + RedisModule_FreeServerInfo(ctx, info); + return REDISMODULE_OK; +} + +int info_gets(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + return info_get(ctx, argv, argc, 's'); +} + +int info_geti(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + return info_get(ctx, argv, argc, 'i'); +} + +int info_getd(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + return info_get(ctx, argv, argc, 'd'); +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -37,5 +82,12 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_RegisterInfoFunc(ctx, InfoFunc) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"info.gets", info_gets,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"info.geti", info_geti,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"info.getd", info_getd,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 659ee79d7..a42681345 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -10,6 +10,27 @@ proc field {info property} { start_server {tags {"modules"}} { r module load $testmodule log-key 0 + test {module reading info} { + # check string, integer and float fields + assert_equal [r info.gets replication role] "master" + assert_equal [r info.geti stats expired_keys] 0 + assert_equal [r info.getd stats expired_stale_perc] 0 + + # the above are always 0, try module info that is non-zero + assert_equal [r info.geti infotest_italian infotest_due] 2 + set tre [r info.getd infotest_italian infotest_tre] + assert {$tre > 3.2 && $tre < 3.4 } + + # search using the wrong section + catch { [r info.gets badname redis_version] } e + assert_match {*not found*} $e + + # check that section filter works + assert { [string match "*usec_per_call*" [r info.gets all cmdstat_info.gets] ] } + catch { [r info.gets default cmdstat_info.gets] ] } e + assert_match {*not found*} $e + } + test {module info all} { set info [r info all] # info all does not contain modules From 04df0cc328c69554e4e6a8b9c2f89305561aa14c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 3 Nov 2019 16:42:31 +0200 Subject: [PATCH 244/320] Module API for loading and saving long double looks like each platform implements long double differently (different bit count) so we can't save them as binary, and we also want to avoid creating a new RDB format version, so we save these are hex strings using "%La". This commit includes a change in the arguments of ld2string to support this. as well as tests for coverage and short reads. coded by @guybe7 --- src/module.c | 27 ++++++++++++++++++ src/networking.c | 2 +- src/object.c | 2 +- src/redismodule.h | 4 +++ src/t_hash.c | 2 +- src/util.c | 62 ++++++++++++++++++++++++----------------- src/util.h | 9 +++++- tests/modules/testrdb.c | 9 +++++- 8 files changed, 87 insertions(+), 30 deletions(-) diff --git a/src/module.c b/src/module.c index f9f654b42..68d1d023d 100644 --- a/src/module.c +++ b/src/module.c @@ -3716,6 +3716,31 @@ loaderr: return 0; } +/* In the context of the rdb_save method of a module data type, saves a long double + * value to the RDB file. The double can be a valid number, a NaN or infinity. + * It is possible to load back the value with RedisModule_LoadLongDouble(). */ +void RM_SaveLongDouble(RedisModuleIO *io, long double value) { + if (io->error) return; + char buf[MAX_LONG_DOUBLE_CHARS]; + /* Long double has different number of bits in different platforms, so we + * save it as a string type. */ + size_t len = ld2string(buf,sizeof(buf),value,LD_STR_HEX); + RM_SaveStringBuffer(io,buf,len+1); /* len+1 for '\0' */ +} + +/* In the context of the rdb_save method of a module data type, loads back the + * long double value saved by RedisModule_SaveLongDouble(). */ +long double RM_LoadLongDouble(RedisModuleIO *io) { + if (io->error) return 0; + long double value; + size_t len; + char* str = RM_LoadStringBuffer(io,&len); + if (!str) return 0; + string2ld(str,len,&value); + RM_Free(str); + return value; +} + /* Iterate over modules, and trigger rdb aux saving for the ones modules types * who asked for it. */ ssize_t rdbSaveModulesAux(rio *rdb, int when) { @@ -6669,6 +6694,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(LoadDouble); REGISTER_API(SaveFloat); REGISTER_API(LoadFloat); + REGISTER_API(SaveLongDouble); + REGISTER_API(LoadLongDouble); REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); diff --git a/src/networking.c b/src/networking.c index e7cc561fa..428ab14ce 100644 --- a/src/networking.c +++ b/src/networking.c @@ -530,7 +530,7 @@ void addReplyHumanLongDouble(client *c, long double d) { decrRefCount(o); } else { char buf[MAX_LONG_DOUBLE_CHARS]; - int len = ld2string(buf,sizeof(buf),d,1); + int len = ld2string(buf,sizeof(buf),d,LD_STR_HUMAN); addReplyProto(c,",",1); addReplyProto(c,buf,len); addReplyProto(c,"\r\n",2); diff --git a/src/object.c b/src/object.c index 70022f897..53ad518a9 100644 --- a/src/object.c +++ b/src/object.c @@ -178,7 +178,7 @@ robj *createStringObjectFromLongLongForValue(long long value) { * The 'humanfriendly' option is used for INCRBYFLOAT and HINCRBYFLOAT. */ robj *createStringObjectFromLongDouble(long double value, int humanfriendly) { char buf[MAX_LONG_DOUBLE_CHARS]; - int len = ld2string(buf,sizeof(buf),value,humanfriendly); + int len = ld2string(buf,sizeof(buf),value,humanfriendly? LD_STR_HUMAN: LD_STR_AUTO); return createStringObject(buf,len); } diff --git a/src/redismodule.h b/src/redismodule.h index ea0d6a139..54d198592 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -457,6 +457,8 @@ void REDISMODULE_API_FUNC(RedisModule_SaveDouble)(RedisModuleIO *io, double valu double REDISMODULE_API_FUNC(RedisModule_LoadDouble)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value); float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); +void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long double value); +long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); @@ -658,6 +660,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(LoadDouble); REDISMODULE_GET_API(SaveFloat); REDISMODULE_GET_API(LoadFloat); + REDISMODULE_GET_API(SaveLongDouble); + REDISMODULE_GET_API(LoadLongDouble); REDISMODULE_GET_API(EmitAOF); REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); diff --git a/src/t_hash.c b/src/t_hash.c index e6ed33819..b9f0db7fc 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -621,7 +621,7 @@ void hincrbyfloatCommand(client *c) { } char buf[MAX_LONG_DOUBLE_CHARS]; - int len = ld2string(buf,sizeof(buf),value,1); + int len = ld2string(buf,sizeof(buf),value,LD_STR_HUMAN); new = sdsnewlen(buf,len); hashTypeSet(o,c->argv[2]->ptr,new,HASH_SET_TAKE_VALUE); addReplyBulkCBuffer(c,buf,len); diff --git a/src/util.c b/src/util.c index 783bcf83b..062a572d4 100644 --- a/src/util.c +++ b/src/util.c @@ -510,15 +510,17 @@ int d2string(char *buf, size_t len, double value) { return len; } -/* Convert a long double into a string. If humanfriendly is non-zero - * it does not use exponential format and trims trailing zeroes at the end, - * however this results in loss of precision. Otherwise exp format is used - * and the output of snprintf() is not modified. +/* Create a string object from a long double. + * If mode is humanfriendly it does not use exponential format and trims trailing + * zeroes at the end (may result in loss of precision). + * If mode is default exp format is used and the output of snprintf() + * is not modified (may result in loss of precision). + * If mode is hex hexadecimal format is used (no loss of precision) * * The function returns the length of the string or zero if there was not * enough buffer room to store it. */ -int ld2string(char *buf, size_t len, long double value, int humanfriendly) { - size_t l; +int ld2string(char *buf, size_t len, long double value, ld2string_mode mode) { + size_t l = 0; if (isinf(value)) { /* Libc in odd systems (Hi Solaris!) will format infinite in a @@ -531,26 +533,36 @@ int ld2string(char *buf, size_t len, long double value, int humanfriendly) { memcpy(buf,"-inf",4); l = 4; } - } else if (humanfriendly) { - /* We use 17 digits precision since with 128 bit floats that precision - * after rounding is able to represent most small decimal numbers in a - * way that is "non surprising" for the user (that is, most small - * decimal numbers will be represented in a way that when converted - * back into a string are exactly the same as what the user typed.) */ - l = snprintf(buf,len,"%.17Lf", value); - if (l+1 > len) return 0; /* No room. */ - /* Now remove trailing zeroes after the '.' */ - if (strchr(buf,'.') != NULL) { - char *p = buf+l-1; - while(*p == '0') { - p--; - l--; - } - if (*p == '.') l--; - } } else { - l = snprintf(buf,len,"%.17Lg", value); - if (l+1 > len) return 0; /* No room. */ + switch (mode) { + case LD_STR_AUTO: + l = snprintf(buf,len,"%.17Lg",value); + if (l+1 > len) return 0; /* No room. */ + break; + case LD_STR_HEX: + l = snprintf(buf,len,"%La",value); + if (l+1 > len) return 0; /* No room. */ + break; + case LD_STR_HUMAN: + /* We use 17 digits precision since with 128 bit floats that precision + * after rounding is able to represent most small decimal numbers in a + * way that is "non surprising" for the user (that is, most small + * decimal numbers will be represented in a way that when converted + * back into a string are exactly the same as what the user typed.) */ + l = snprintf(buf,len,"%.17Lf",value); + if (l+1 > len) return 0; /* No room. */ + /* Now remove trailing zeroes after the '.' */ + if (strchr(buf,'.') != NULL) { + char *p = buf+l-1; + while(*p == '0') { + p--; + l--; + } + if (*p == '.') l--; + } + break; + default: return 0; /* Invalid mode. */ + } } buf[l] = '\0'; return l; diff --git a/src/util.h b/src/util.h index b6c01aa59..a91addb80 100644 --- a/src/util.h +++ b/src/util.h @@ -38,6 +38,13 @@ * This should be the size of the buffer given to ld2string */ #define MAX_LONG_DOUBLE_CHARS 5*1024 +/* long double to string convertion options */ +typedef enum { + LD_STR_AUTO, /* %.17Lg */ + LD_STR_HUMAN, /* %.17Lf + Trimming of trailing zeros */ + LD_STR_HEX /* %La */ +} ld2string_mode; + int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase); int stringmatch(const char *p, const char *s, int nocase); int stringmatchlen_fuzz_test(void); @@ -49,7 +56,7 @@ int string2ll(const char *s, size_t slen, long long *value); int string2l(const char *s, size_t slen, long *value); int string2ld(const char *s, size_t slen, long double *dp); int d2string(char *buf, size_t len, double value); -int ld2string(char *buf, size_t len, long double value, int humanfriendly); +int ld2string(char *buf, size_t len, long double value, ld2string_mode mode); sds getAbsolutePath(char *filename); unsigned long getTimeZone(void); int pathIsBaseName(char *path); diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index eb8d1a999..8a262e8a7 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -15,11 +15,16 @@ RedisModuleString *after_str = NULL; void *testrdb_type_load(RedisModuleIO *rdb, int encver) { int count = RedisModule_LoadSigned(rdb); + RedisModuleString *str = RedisModule_LoadString(rdb); + float f = RedisModule_LoadFloat(rdb); + long double ld = RedisModule_LoadLongDouble(rdb); if (RedisModule_IsIOError(rdb)) return NULL; + /* Using the values only after checking for io errors. */ assert(count==1); assert(encver==1); - RedisModuleString *str = RedisModule_LoadString(rdb); + assert(f==1.5f); + assert(ld==0.333333333333333333L); return str; } @@ -27,6 +32,8 @@ void testrdb_type_save(RedisModuleIO *rdb, void *value) { RedisModuleString *str = (RedisModuleString*)value; RedisModule_SaveSigned(rdb, 1); RedisModule_SaveString(rdb, str); + RedisModule_SaveFloat(rdb, 1.5); + RedisModule_SaveLongDouble(rdb, 0.333333333333333333L); } void testrdb_aux_save(RedisModuleIO *rdb, int when) { From 9cd39978518e8fd31b1b76fd41d9605d275fc2ed Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 3 Nov 2019 17:35:35 +0200 Subject: [PATCH 245/320] Module API for PUBLISH, FLUSHALL, RANDOMKEY, DBSIZE --- src/db.c | 44 ++++++++++++++++++++++++-------------------- src/module.c | 35 +++++++++++++++++++++++++++++++++++ src/redismodule.h | 8 ++++++++ src/server.h | 2 ++ 4 files changed, 69 insertions(+), 20 deletions(-) diff --git a/src/db.c b/src/db.c index 2c0a0cdd3..ad19b42dd 100644 --- a/src/db.c +++ b/src/db.c @@ -461,6 +461,29 @@ int getFlushCommandFlags(client *c, int *flags) { return C_OK; } +/* Flushes the whole server data set. */ +void flushAllDataAndResetRDB(int flags) { + server.dirty += emptyDb(-1,flags,NULL); + if (server.rdb_child_pid != -1) killRDBChild(); + if (server.saveparamslen > 0) { + /* Normally rdbSave() will reset dirty, but we don't want this here + * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ + int saved_dirty = server.dirty; + rdbSaveInfo rsi, *rsiptr; + rsiptr = rdbPopulateSaveInfo(&rsi); + rdbSave(server.rdb_filename,rsiptr); + server.dirty = saved_dirty; + } + server.dirty++; +#if defined(USE_JEMALLOC) + /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. + * for large databases, flushdb blocks for long anyway, so a bit more won't + * harm and this way the flush and purge will be synchroneus. */ + if (!(flags & EMPTYDB_ASYNC)) + jemalloc_purge(); +#endif +} + /* FLUSHDB [ASYNC] * * Flushes the currently SELECTed Redis DB. */ @@ -484,28 +507,9 @@ void flushdbCommand(client *c) { * Flushes the whole server data set. */ void flushallCommand(client *c) { int flags; - if (getFlushCommandFlags(c,&flags) == C_ERR) return; - server.dirty += emptyDb(-1,flags,NULL); + flushAllDataAndResetRDB(flags); addReply(c,shared.ok); - if (server.rdb_child_pid != -1) killRDBChild(); - if (server.saveparamslen > 0) { - /* Normally rdbSave() will reset dirty, but we don't want this here - * as otherwise FLUSHALL will not be replicated nor put into the AOF. */ - int saved_dirty = server.dirty; - rdbSaveInfo rsi, *rsiptr; - rsiptr = rdbPopulateSaveInfo(&rsi); - rdbSave(server.rdb_filename,rsiptr); - server.dirty = saved_dirty; - } - server.dirty++; -#if defined(USE_JEMALLOC) - /* jemalloc 5 doesn't release pages back to the OS when there's no traffic. - * for large databases, flushdb blocks for long anyway, so a bit more won't - * harm and this way the flush and purge will be synchroneus. */ - if (!(flags & EMPTYDB_ASYNC)) - jemalloc_purge(); -#endif } /* This command implements DEL and LAZYDEL. */ diff --git a/src/module.c b/src/module.c index f9f654b42..5453aed47 100644 --- a/src/module.c +++ b/src/module.c @@ -1677,6 +1677,15 @@ int RM_GetClientInfoById(void *ci, uint64_t id) { return modulePopulateClientInfoStructure(ci,client,structver); } +/* Publish a message to subscribers (see PUBLISH command). */ +int RM_PublishMessage(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message) { + UNUSED(ctx); + int receivers = pubsubPublishMessage(channel, message); + if (server.cluster_enabled) + clusterPropagatePublish(channel, message); + return receivers; +} + /* Return the currently selected DB. */ int RM_GetSelectedDb(RedisModuleCtx *ctx) { return ctx->client->db->id; @@ -1964,6 +1973,28 @@ int RM_SetExpire(RedisModuleKey *key, mstime_t expire) { return REDISMODULE_OK; } +/* Performs similar operation to FLUSHALL, and optionally start a new AOF file (if enabled) + * If restart_aof is true, you must make sure the command that triggered this call is not + * propagated to the AOF file. + * When async is set to true, db contents will be freed by a background thread. */ +void RM_ResetDataset(int restart_aof, int async) { + if (restart_aof && server.aof_state != AOF_OFF) stopAppendOnly(); + flushAllDataAndResetRDB(async? EMPTYDB_ASYNC: EMPTYDB_NO_FLAGS); + if (server.aof_enabled && restart_aof) restartAOFAfterSYNC(); +} + +/* Returns the number of keys in the current db. */ +unsigned long long RM_DbSize(RedisModuleCtx *ctx) { + return dictSize(ctx->client->db->dict); +} + +/* Returns a name of a random key, or NULL if current db is empty. */ +RedisModuleString *RM_RandomKey(RedisModuleCtx *ctx) { + robj *key = dbRandomKey(ctx->client->db); + autoMemoryAdd(ctx,REDISMODULE_AM_STRING,key); + return key; +} + /* -------------------------------------------------------------------------- * Key API for String type * -------------------------------------------------------------------------- */ @@ -6630,6 +6661,9 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(StringTruncate); REGISTER_API(SetExpire); REGISTER_API(GetExpire); + REGISTER_API(ResetDataset); + REGISTER_API(DbSize); + REGISTER_API(RandomKey); REGISTER_API(ZsetAdd); REGISTER_API(ZsetIncrby); REGISTER_API(ZsetScore); @@ -6757,6 +6791,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(InfoAddFieldLongLong); REGISTER_API(InfoAddFieldULongLong); REGISTER_API(GetClientInfoById); + REGISTER_API(PublishMessage); REGISTER_API(SubscribeToServerEvent); REGISTER_API(BlockClientOnKeys); REGISTER_API(SignalKeyAsReady); diff --git a/src/redismodule.h b/src/redismodule.h index ea0d6a139..77019f89e 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -416,6 +416,9 @@ char *REDISMODULE_API_FUNC(RedisModule_StringDMA)(RedisModuleKey *key, size_t *l int REDISMODULE_API_FUNC(RedisModule_StringTruncate)(RedisModuleKey *key, size_t newlen); mstime_t REDISMODULE_API_FUNC(RedisModule_GetExpire)(RedisModuleKey *key); int REDISMODULE_API_FUNC(RedisModule_SetExpire)(RedisModuleKey *key, mstime_t expire); +void REDISMODULE_API_FUNC(RedisModule_ResetDataset)(int restart_aof, int async); +unsigned long long REDISMODULE_API_FUNC(RedisModule_DbSize)(RedisModuleCtx *ctx); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_RandomKey)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ZsetAdd)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr); int REDISMODULE_API_FUNC(RedisModule_ZsetIncrby)(RedisModuleKey *key, double score, RedisModuleString *ele, int *flagsptr, double *newscore); int REDISMODULE_API_FUNC(RedisModule_ZsetScore)(RedisModuleKey *key, RedisModuleString *ele, double *score); @@ -435,6 +438,7 @@ int REDISMODULE_API_FUNC(RedisModule_IsKeysPositionRequest)(RedisModuleCtx *ctx) void REDISMODULE_API_FUNC(RedisModule_KeyAtPos)(RedisModuleCtx *ctx, int pos); unsigned long long REDISMODULE_API_FUNC(RedisModule_GetClientId)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_GetClientInfoById)(void *ci, uint64_t id); +int REDISMODULE_API_FUNC(RedisModule_PublishMessage)(RedisModuleCtx *ctx, RedisModuleString *channel, RedisModuleString *message); int REDISMODULE_API_FUNC(RedisModule_GetContextFlags)(RedisModuleCtx *ctx); void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods); @@ -619,6 +623,9 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(StringTruncate); REDISMODULE_GET_API(GetExpire); REDISMODULE_GET_API(SetExpire); + REDISMODULE_GET_API(ResetDataset); + REDISMODULE_GET_API(DbSize); + REDISMODULE_GET_API(RandomKey); REDISMODULE_GET_API(ZsetAdd); REDISMODULE_GET_API(ZsetIncrby); REDISMODULE_GET_API(ZsetScore); @@ -705,6 +712,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(InfoAddFieldLongLong); REDISMODULE_GET_API(InfoAddFieldULongLong); REDISMODULE_GET_API(GetClientInfoById); + REDISMODULE_GET_API(PublishMessage); REDISMODULE_GET_API(SubscribeToServerEvent); REDISMODULE_GET_API(BlockClientOnKeys); REDISMODULE_GET_API(SignalKeyAsReady); diff --git a/src/server.h b/src/server.h index f724f7d64..4287d1adf 100644 --- a/src/server.h +++ b/src/server.h @@ -1862,6 +1862,7 @@ void aofRewriteBufferReset(void); unsigned long aofRewriteBufferSize(void); ssize_t aofReadDiffFromParent(void); void killAppendOnlyChild(void); +void restartAOFAfterSYNC(); /* Child info */ void openChildInfoPipe(void); @@ -2101,6 +2102,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o); #define EMPTYDB_ASYNC (1<<0) /* Reclaim memory in another thread. */ long long emptyDb(int dbnum, int flags, void(callback)(void*)); long long emptyDbGeneric(redisDb *dbarray, int dbnum, int flags, void(callback)(void*)); +void flushAllDataAndResetRDB(int flags); long long dbTotalServerKeyCount(); int selectDb(client *c, int id); From 8541c6a0588e2bca94a8b44fbbeafd202c622cf5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 4 Nov 2019 07:57:52 +0200 Subject: [PATCH 246/320] Adding RM_ServerInfoGetFieldC --- src/module.c | 14 ++++++++++---- src/redismodule.h | 6 ++++-- tests/modules/infotest.c | 14 ++++++++++++-- tests/unit/moduleapi/infotest.tcl | 1 + 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index ce85c479d..fea66f293 100644 --- a/src/module.c +++ b/src/module.c @@ -5534,11 +5534,17 @@ RedisModuleString *RM_ServerInfoGetField(RedisModuleCtx *ctx, RedisModuleServerI return o; } +/* Similar to RM_ServerInfoGetField, but returns a char* which should not be freed but the caller. */ +const char *RM_ServerInfoGetFieldC(RedisModuleServerInfoData *data, const char* field) { + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) return NULL; + return val; +} + /* Get the value of a field from data collected with RM_GetServerInfo(). If the * field is not found, or is not numerical, return value will be 0, and the * optional out_err argument will be set to REDISMODULE_ERR. */ -long long RM_ServerInfoGetFieldNumerical(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err) { - UNUSED(ctx); +long long RM_ServerInfoGetFieldNumerical(RedisModuleServerInfoData *data, const char* field, int *out_err) { long long ll; sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); if (val == raxNotFound) { @@ -5556,8 +5562,7 @@ long long RM_ServerInfoGetFieldNumerical(RedisModuleCtx *ctx, RedisModuleServerI /* Get the value of a field from data collected with RM_GetServerInfo(). If the * field is not found, or is not a double, return value will be 0, and the * optional out_err argument will be set to REDISMODULE_ERR. */ -double RM_ServerInfoGetFieldDouble(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err) { - UNUSED(ctx); +double RM_ServerInfoGetFieldDouble(RedisModuleServerInfoData *data, const char* field, int *out_err) { double dbl; sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); if (val == raxNotFound) { @@ -6862,6 +6867,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetServerInfo); REGISTER_API(FreeServerInfo); REGISTER_API(ServerInfoGetField); + REGISTER_API(ServerInfoGetFieldC); REGISTER_API(ServerInfoGetFieldNumerical); REGISTER_API(ServerInfoGetFieldDouble); REGISTER_API(GetClientInfoById); diff --git a/src/redismodule.h b/src/redismodule.h index d923a1b8a..0c289848f 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -506,8 +506,9 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx * RedisModuleServerInfoData *REDISMODULE_API_FUNC(RedisModule_GetServerInfo)(RedisModuleCtx *ctx, const char *section); void REDISMODULE_API_FUNC(RedisModule_FreeServerInfo)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetField)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field); -long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldNumerical)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err); -double REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldDouble)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field, int *out_err); +const char *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldC)(RedisModuleServerInfoData *data, const char* field); +long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldNumerical)(RedisModuleServerInfoData *data, const char* field, int *out_err); +double REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldDouble)(RedisModuleServerInfoData *data, const char* field, int *out_err); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); @@ -713,6 +714,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetServerInfo); REDISMODULE_GET_API(FreeServerInfo); REDISMODULE_GET_API(ServerInfoGetField); + REDISMODULE_GET_API(ServerInfoGetFieldC); REDISMODULE_GET_API(ServerInfoGetFieldNumerical); REDISMODULE_GET_API(ServerInfoGetFieldDouble); REDISMODULE_GET_API(GetClientInfoById); diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index 460ebb9c0..a21fefffc 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -41,13 +41,17 @@ int info_get(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, char field field = RedisModule_StringPtrLen(argv[2], NULL); RedisModuleServerInfoData *info = RedisModule_GetServerInfo(ctx, section); if (field_type=='i') { - long long ll = RedisModule_ServerInfoGetFieldNumerical(ctx, info, field, &err); + long long ll = RedisModule_ServerInfoGetFieldNumerical(info, field, &err); if (err==REDISMODULE_OK) RedisModule_ReplyWithLongLong(ctx, ll); } else if (field_type=='d') { - double d = RedisModule_ServerInfoGetFieldDouble(ctx, info, field, &err); + double d = RedisModule_ServerInfoGetFieldDouble(info, field, &err); if (err==REDISMODULE_OK) RedisModule_ReplyWithDouble(ctx, d); + } else if (field_type=='c') { + const char *str = RedisModule_ServerInfoGetFieldC(info, field); + if (str) + RedisModule_ReplyWithCString(ctx, str); } else { RedisModuleString *str = RedisModule_ServerInfoGetField(ctx, info, field); if (str) { @@ -66,6 +70,10 @@ int info_gets(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return info_get(ctx, argv, argc, 's'); } +int info_getc(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + return info_get(ctx, argv, argc, 'c'); +} + int info_geti(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return info_get(ctx, argv, argc, 'i'); } @@ -84,6 +92,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) if (RedisModule_CreateCommand(ctx,"info.gets", info_gets,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"info.getc", info_getc,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"info.geti", info_geti,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"info.getd", info_getd,"",0,0,0) == REDISMODULE_ERR) diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index a42681345..225798dd5 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -13,6 +13,7 @@ start_server {tags {"modules"}} { test {module reading info} { # check string, integer and float fields assert_equal [r info.gets replication role] "master" + assert_equal [r info.getc replication role] "master" assert_equal [r info.geti stats expired_keys] 0 assert_equal [r info.getd stats expired_stale_perc] 0 From ba1478c9f933dfaf98b079e26ff096378c0a8f4e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 4 Nov 2019 08:50:29 +0200 Subject: [PATCH 247/320] Add RM_ServerInfoGetFieldUnsigned rename RM_ServerInfoGetFieldNumerical RM_ServerInfoGetFieldSigned move string2ull to util.c fix leak in RM_GetServerInfo when duplicate info fields exist --- src/module.c | 30 +++++++++++++++++++++++++----- src/redismodule.h | 6 ++++-- src/t_stream.c | 20 -------------------- src/util.c | 20 ++++++++++++++++++++ src/util.h | 1 + tests/modules/infotest.c | 13 ++++++++++++- tests/unit/moduleapi/infotest.tcl | 4 ++++ 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/src/module.c b/src/module.c index fea66f293..2644883c9 100644 --- a/src/module.c +++ b/src/module.c @@ -5497,7 +5497,8 @@ RedisModuleServerInfoData *RM_GetServerInfo(RedisModuleCtx *ctx, const char *sec unsigned char *key = (unsigned char*)line; size_t keylen = (intptr_t)sep-(intptr_t)line; sds val = sdsnewlen(sep+1,sdslen(line)-((intptr_t)sep-(intptr_t)line)-1); - raxTryInsert(d->rax,key,keylen,val,NULL); + if (!raxTryInsert(d->rax,key,keylen,val,NULL)) + sdsfree(val); } sdsfree(info); sdsfreesplitres(lines,totlines); @@ -5542,9 +5543,9 @@ const char *RM_ServerInfoGetFieldC(RedisModuleServerInfoData *data, const char* } /* Get the value of a field from data collected with RM_GetServerInfo(). If the - * field is not found, or is not numerical, return value will be 0, and the - * optional out_err argument will be set to REDISMODULE_ERR. */ -long long RM_ServerInfoGetFieldNumerical(RedisModuleServerInfoData *data, const char* field, int *out_err) { + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +long long RM_ServerInfoGetFieldSigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { long long ll; sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); if (val == raxNotFound) { @@ -5559,6 +5560,24 @@ long long RM_ServerInfoGetFieldNumerical(RedisModuleServerInfoData *data, const return ll; } +/* Get the value of a field from data collected with RM_GetServerInfo(). If the + * field is not found, or is not numerical or out of range, return value will be + * 0, and the optional out_err argument will be set to REDISMODULE_ERR. */ +unsigned long long RM_ServerInfoGetFieldUnsigned(RedisModuleServerInfoData *data, const char* field, int *out_err) { + unsigned long long ll; + sds val = raxFind(data->rax, (unsigned char *)field, strlen(field)); + if (val == raxNotFound) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (!string2ull(val,&ll)) { + if (out_err) *out_err = REDISMODULE_ERR; + return 0; + } + if (out_err) *out_err = REDISMODULE_OK; + return ll; +} + /* Get the value of a field from data collected with RM_GetServerInfo(). If the * field is not found, or is not a double, return value will be 0, and the * optional out_err argument will be set to REDISMODULE_ERR. */ @@ -6868,7 +6887,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(FreeServerInfo); REGISTER_API(ServerInfoGetField); REGISTER_API(ServerInfoGetFieldC); - REGISTER_API(ServerInfoGetFieldNumerical); + REGISTER_API(ServerInfoGetFieldSigned); + REGISTER_API(ServerInfoGetFieldUnsigned); REGISTER_API(ServerInfoGetFieldDouble); REGISTER_API(GetClientInfoById); REGISTER_API(SubscribeToServerEvent); diff --git a/src/redismodule.h b/src/redismodule.h index 0c289848f..fd6dc77ce 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -507,7 +507,8 @@ RedisModuleServerInfoData *REDISMODULE_API_FUNC(RedisModule_GetServerInfo)(Redis void REDISMODULE_API_FUNC(RedisModule_FreeServerInfo)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetField)(RedisModuleCtx *ctx, RedisModuleServerInfoData *data, const char* field); const char *REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldC)(RedisModuleServerInfoData *data, const char* field); -long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldNumerical)(RedisModuleServerInfoData *data, const char* field, int *out_err); +long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldSigned)(RedisModuleServerInfoData *data, const char* field, int *out_err); +unsigned long long REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldUnsigned)(RedisModuleServerInfoData *data, const char* field, int *out_err); double REDISMODULE_API_FUNC(RedisModule_ServerInfoGetFieldDouble)(RedisModuleServerInfoData *data, const char* field, int *out_err); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); @@ -715,7 +716,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(FreeServerInfo); REDISMODULE_GET_API(ServerInfoGetField); REDISMODULE_GET_API(ServerInfoGetFieldC); - REDISMODULE_GET_API(ServerInfoGetFieldNumerical); + REDISMODULE_GET_API(ServerInfoGetFieldSigned); + REDISMODULE_GET_API(ServerInfoGetFieldUnsigned); REDISMODULE_GET_API(ServerInfoGetFieldDouble); REDISMODULE_GET_API(GetClientInfoById); REDISMODULE_GET_API(SubscribeToServerEvent); diff --git a/src/t_stream.c b/src/t_stream.c index ea9a620f1..e6694f0b7 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1070,26 +1070,6 @@ robj *streamTypeLookupWriteOrCreate(client *c, robj *key) { return o; } -/* Helper function to convert a string to an unsigned long long value. - * The function attempts to use the faster string2ll() function inside - * Redis: if it fails, strtoull() is used instead. The function returns - * 1 if the conversion happened successfully or 0 if the number is - * invalid or out of range. */ -int string2ull(const char *s, unsigned long long *value) { - long long ll; - if (string2ll(s,strlen(s),&ll)) { - if (ll < 0) return 0; /* Negative values are out of range. */ - *value = ll; - return 1; - } - errno = 0; - char *endptr = NULL; - *value = strtoull(s,&endptr,10); - if (errno == EINVAL || errno == ERANGE || !(*s != '\0' && *endptr == '\0')) - return 0; /* strtoull() failed. */ - return 1; /* Conversion done! */ -} - /* Parse a stream ID in the format given by clients to Redis, that is * -, and converts it into a streamID structure. If * the specified ID is invalid C_ERR is returned and an error is reported diff --git a/src/util.c b/src/util.c index 6e9dc2117..f0f1a4ed3 100644 --- a/src/util.c +++ b/src/util.c @@ -423,6 +423,26 @@ int string2ll(const char *s, size_t slen, long long *value) { return 1; } +/* Helper function to convert a string to an unsigned long long value. + * The function attempts to use the faster string2ll() function inside + * Redis: if it fails, strtoull() is used instead. The function returns + * 1 if the conversion happened successfully or 0 if the number is + * invalid or out of range. */ +int string2ull(const char *s, unsigned long long *value) { + long long ll; + if (string2ll(s,strlen(s),&ll)) { + if (ll < 0) return 0; /* Negative values are out of range. */ + *value = ll; + return 1; + } + errno = 0; + char *endptr = NULL; + *value = strtoull(s,&endptr,10); + if (errno == EINVAL || errno == ERANGE || !(*s != '\0' && *endptr == '\0')) + return 0; /* strtoull() failed. */ + return 1; /* Conversion done! */ +} + /* Convert a string into a long. Returns 1 if the string could be parsed into a * (non-overflowing) long, 0 otherwise. The value will be set to the parsed * value when appropriate. */ diff --git a/src/util.h b/src/util.h index ab1d71f6c..7e162686e 100644 --- a/src/util.h +++ b/src/util.h @@ -46,6 +46,7 @@ uint32_t digits10(uint64_t v); uint32_t sdigits10(int64_t v); int ll2string(char *s, size_t len, long long value); int string2ll(const char *s, size_t slen, long long *value); +int string2ull(const char *s, unsigned long long *value); int string2l(const char *s, size_t slen, long *value); int string2ld(const char *s, size_t slen, long double *dp); int string2d(const char *s, size_t slen, double *dp); diff --git a/tests/modules/infotest.c b/tests/modules/infotest.c index a21fefffc..4cb77ee87 100644 --- a/tests/modules/infotest.c +++ b/tests/modules/infotest.c @@ -5,6 +5,7 @@ void InfoFunc(RedisModuleInfoCtx *ctx, int for_crash_report) { RedisModule_InfoAddSection(ctx, ""); RedisModule_InfoAddFieldLongLong(ctx, "global", -2); + RedisModule_InfoAddFieldULongLong(ctx, "uglobal", (unsigned long long)-2); RedisModule_InfoAddSection(ctx, "Spanish"); RedisModule_InfoAddFieldCString(ctx, "uno", "one"); @@ -41,7 +42,11 @@ int info_get(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, char field field = RedisModule_StringPtrLen(argv[2], NULL); RedisModuleServerInfoData *info = RedisModule_GetServerInfo(ctx, section); if (field_type=='i') { - long long ll = RedisModule_ServerInfoGetFieldNumerical(info, field, &err); + long long ll = RedisModule_ServerInfoGetFieldSigned(info, field, &err); + if (err==REDISMODULE_OK) + RedisModule_ReplyWithLongLong(ctx, ll); + } else if (field_type=='u') { + unsigned long long ll = (unsigned long long)RedisModule_ServerInfoGetFieldUnsigned(info, field, &err); if (err==REDISMODULE_OK) RedisModule_ReplyWithLongLong(ctx, ll); } else if (field_type=='d') { @@ -78,6 +83,10 @@ int info_geti(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return info_get(ctx, argv, argc, 'i'); } +int info_getu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + return info_get(ctx, argv, argc, 'u'); +} + int info_getd(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return info_get(ctx, argv, argc, 'd'); } @@ -96,6 +105,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"info.geti", info_geti,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"info.getu", info_getu,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"info.getd", info_getd,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; diff --git a/tests/unit/moduleapi/infotest.tcl b/tests/unit/moduleapi/infotest.tcl index 225798dd5..80a28656c 100644 --- a/tests/unit/moduleapi/infotest.tcl +++ b/tests/unit/moduleapi/infotest.tcl @@ -17,6 +17,10 @@ start_server {tags {"modules"}} { assert_equal [r info.geti stats expired_keys] 0 assert_equal [r info.getd stats expired_stale_perc] 0 + # check signed and unsigned + assert_equal [r info.geti infotest infotest_global] -2 + assert_equal [r info.getu infotest infotest_uglobal] -2 + # the above are always 0, try module info that is non-zero assert_equal [r info.geti infotest_italian infotest_due] 2 set tre [r info.getd infotest_italian infotest_tre] From 4f630e931b7fd165371c259e165a382da291fde9 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Thu, 29 Mar 2018 17:46:13 +0700 Subject: [PATCH 248/320] Modules: Test RedisModule_BlockClientOnKeys --- runtest-moduleapi | 1 + tests/modules/Makefile | 3 +- tests/modules/blockonkeys.c | 261 +++++++++++++++++++++++++++ tests/unit/moduleapi/blockonkeys.tcl | 85 +++++++++ 4 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 tests/modules/blockonkeys.c create mode 100644 tests/unit/moduleapi/blockonkeys.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 9301002c9..e48535126 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -21,4 +21,5 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/propagate \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ +--single unit/moduleapi/blockonkeys \ "${@}" diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 71c0b5ef8..9e27758a2 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -18,7 +18,8 @@ TEST_MODULES = \ infotest.so \ propagate.so \ misc.so \ - hooks.so + hooks.so \ + blockonkeys.so .PHONY: all diff --git a/tests/modules/blockonkeys.c b/tests/modules/blockonkeys.c new file mode 100644 index 000000000..959918b1c --- /dev/null +++ b/tests/modules/blockonkeys.c @@ -0,0 +1,261 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include + +#define LIST_SIZE 1024 + +typedef struct { + long long list[LIST_SIZE]; + long long length; +} fsl_t; /* Fixed-size list */ + +static RedisModuleType *fsltype = NULL; + +fsl_t *fsl_type_create() { + fsl_t *o; + o = RedisModule_Alloc(sizeof(*o)); + o->length = 0; + return o; +} + +void fsl_type_free(fsl_t *o) { + RedisModule_Free(o); +} + +/* ========================== "fsltype" type methods ======================= */ + +void *fsl_rdb_load(RedisModuleIO *rdb, int encver) { + if (encver != 0) { + return NULL; + } + fsl_t *fsl = fsl_type_create(); + fsl->length = RedisModule_LoadUnsigned(rdb); + for (long long i = 0; i < fsl->length; i++) + fsl->list[i] = RedisModule_LoadSigned(rdb); + return fsl; +} + +void fsl_rdb_save(RedisModuleIO *rdb, void *value) { + fsl_t *fsl = value; + RedisModule_SaveUnsigned(rdb,fsl->length); + for (long long i = 0; i < fsl->length; i++) + RedisModule_SaveSigned(rdb, fsl->list[i]); +} + +void fsl_aofrw(RedisModuleIO *aof, RedisModuleString *key, void *value) { + fsl_t *fsl = value; + for (long long i = 0; i < fsl->length; i++) + RedisModule_EmitAOF(aof, "FSL.PUSH","sl", key, fsl->list[i]); +} + +void fsl_free(void *value) { + fsl_type_free(value); +} + +/* ========================== helper methods ======================= */ + +int get_fsl(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode, int create, fsl_t **fsl, int reply_on_failure) { + RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, mode); + + int type = RedisModule_KeyType(key); + if (type != REDISMODULE_KEYTYPE_EMPTY && RedisModule_ModuleTypeGetType(key) != fsltype) { + RedisModule_CloseKey(key); + if (reply_on_failure) + RedisModule_ReplyWithError(ctx, REDISMODULE_ERRORMSG_WRONGTYPE); + return 0; + } + + /* Create an empty value object if the key is currently empty. */ + if (type == REDISMODULE_KEYTYPE_EMPTY) { + if (!create) { + /* Key is empty but we cannot create */ + RedisModule_CloseKey(key); + *fsl = NULL; + return 1; + } + *fsl = fsl_type_create(); + RedisModule_ModuleTypeSetValue(key, fsltype, *fsl); + } else { + *fsl = RedisModule_ModuleTypeGetValue(key); + } + + RedisModule_CloseKey(key); + return 1; +} + +/* ========================== commands ======================= */ + +/* FSL.PUSH - Push an integer to the fixed-size list (to the right). + * It must be greater than the element in the head of the list. */ +int fsl_push(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 3) + return RedisModule_WrongArity(ctx); + + long long ele; + if (RedisModule_StringToLongLong(argv[2],&ele) != REDISMODULE_OK) + return RedisModule_ReplyWithError(ctx,"ERR invalid integer"); + + fsl_t *fsl; + if (!get_fsl(ctx, argv[1], REDISMODULE_WRITE, 1, &fsl, 1)) + return REDISMODULE_OK; + + if (fsl->length == LIST_SIZE) + return RedisModule_ReplyWithError(ctx,"ERR list is full"); + + if (fsl->length != 0 && fsl->list[fsl->length-1] >= ele) + return RedisModule_ReplyWithError(ctx,"ERR new element has to be greater than the head element"); + + fsl->list[fsl->length++] = ele; + + if (fsl->length >= 2) + RedisModule_SignalKeyAsReady(ctx, argv[1]); + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} + +int bpop2_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); + + fsl_t *fsl; + if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) + return REDISMODULE_ERR; + + if (!fsl || fsl->length < 2) + return REDISMODULE_ERR; + + RedisModule_ReplyWithArray(ctx, 2); + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + return REDISMODULE_OK; +} + +int bpop2_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); +} + + +/* FSL.BPOP2 - Block clients until list has two or more elements. + * When that happens, unblock client and pop the last two elements (from the right). */ +int fsl_bpop2(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 3) + return RedisModule_WrongArity(ctx); + + long long timeout; + if (RedisModule_StringToLongLong(argv[2],&timeout) != REDISMODULE_OK || timeout < 0) + return RedisModule_ReplyWithError(ctx,"ERR invalid timeout"); + + fsl_t *fsl; + if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1)) + return REDISMODULE_OK; + + if (!fsl || fsl->length < 2) { + /* Key is empty or has <2 elements, we must block */ + RedisModule_BlockClientOnKeys(ctx, bpop2_reply_callback, bpop2_timeout_callback, + NULL, timeout, &argv[1], 1, NULL); + } else { + RedisModule_ReplyWithArray(ctx, 2); + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + } + + return REDISMODULE_OK; +} + +int bpopgt_reply_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleString *keyname = RedisModule_GetBlockedClientReadyKey(ctx); + long long gt = (long long)RedisModule_GetBlockedClientPrivateData(ctx); + + fsl_t *fsl; + if (!get_fsl(ctx, keyname, REDISMODULE_READ, 0, &fsl, 0)) + return REDISMODULE_ERR; + + if (!fsl || fsl->list[fsl->length-1] <= gt) + return REDISMODULE_ERR; + + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + return REDISMODULE_OK; +} + +int bpopgt_timeout_callback(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + return RedisModule_ReplyWithSimpleString(ctx, "Request timedout"); +} + +void bpopgt_free_privdata(RedisModuleCtx *ctx, void *privdata) { + /* Nothing to do because privdata is actually a 'long long', + * not a pointer to the heap */ + REDISMODULE_NOT_USED(ctx); + REDISMODULE_NOT_USED(privdata); +} + +/* FSL.BPOPGT - Block clients until list has an element greater than . + * When that happens, unblock client and pop the last element (from the right). */ +int fsl_bpopgt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 4) + return RedisModule_WrongArity(ctx); + + long long gt; + if (RedisModule_StringToLongLong(argv[2],>) != REDISMODULE_OK) + return RedisModule_ReplyWithError(ctx,"ERR invalid integer"); + + long long timeout; + if (RedisModule_StringToLongLong(argv[3],&timeout) != REDISMODULE_OK || timeout < 0) + return RedisModule_ReplyWithError(ctx,"ERR invalid timeout"); + + fsl_t *fsl; + if (!get_fsl(ctx, argv[1], REDISMODULE_READ, 0, &fsl, 1)) + return REDISMODULE_OK; + + if (!fsl || fsl->list[fsl->length-1] <= gt) { + /* Key is empty or has <2 elements, we must block */ + RedisModule_BlockClientOnKeys(ctx, bpopgt_reply_callback, bpopgt_timeout_callback, + bpopgt_free_privdata, timeout, &argv[1], 1, (void*)gt); + } else { + RedisModule_ReplyWithLongLong(ctx, fsl->list[--fsl->length]); + } + + return REDISMODULE_OK; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx, "blockonkeys", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModuleTypeMethods tm = { + .version = REDISMODULE_TYPE_METHOD_VERSION, + .rdb_load = fsl_rdb_load, + .rdb_save = fsl_rdb_save, + .aof_rewrite = fsl_aofrw, + .mem_usage = NULL, + .free = fsl_free, + .digest = NULL + }; + + fsltype = RedisModule_CreateDataType(ctx, "fsltype_t", 0, &tm); + if (fsltype == NULL) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fsl.push",fsl_push,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fsl.bpop2",fsl_bpop2,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"fsl.bpopgt",fsl_bpopgt,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/blockonkeys.tcl b/tests/unit/moduleapi/blockonkeys.tcl new file mode 100644 index 000000000..cb99ab1c9 --- /dev/null +++ b/tests/unit/moduleapi/blockonkeys.tcl @@ -0,0 +1,85 @@ +set testmodule [file normalize tests/modules/blockonkeys.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module client blocked on keys (no metadata): No block} { + r del k + r fsl.push k 33 + r fsl.push k 34 + r fsl.bpop2 k 0 + } {34 33} + + test {Module client blocked on keys (no metadata): Timeout} { + r del k + set rd [redis_deferring_client] + r fsl.push k 33 + $rd fsl.bpop2 k 1 + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (no metadata): Blocked, case 1} { + r del k + set rd [redis_deferring_client] + r fsl.push k 33 + $rd fsl.bpop2 k 0 + r fsl.push k 34 + assert_equal {34 33} [$rd read] + } + + test {Module client blocked on keys (no metadata): Blocked, case 2} { + r del k + set rd [redis_deferring_client] + r fsl.push k 33 + r fsl.push k 34 + $rd fsl.bpop2 k 0 + assert_equal {34 33} [$rd read] + } + + test {Module client blocked on keys (with metadata): No block} { + r del k + r fsl.push k 34 + r fsl.bpopgt k 30 0 + } {34} + + test {Module client blocked on keys (with metadata): Timeout} { + r del k + set rd [redis_deferring_client] + r fsl.push k 33 + $rd fsl.bpopgt k 35 1 + assert_equal {Request timedout} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, case 1} { + r del k + set rd [redis_deferring_client] + r fsl.push k 33 + $rd fsl.bpopgt k 33 0 + r fsl.push k 34 + assert_equal {34} [$rd read] + } + + test {Module client blocked on keys (with metadata): Blocked, case 2} { + r del k + set rd [redis_deferring_client] + $rd fsl.bpopgt k 35 0 + r fsl.push k 33 + r fsl.push k 34 + r fsl.push k 35 + r fsl.push k 36 + assert_equal {36} [$rd read] + } + + test {Module client blocked on keys does not wake up on wrong type} { + r del k + set rd [redis_deferring_client] + $rd fsl.bpop2 k 0 + r lpush k 12 + r lpush k 13 + r lpush k 14 + r del k + r fsl.push k 33 + r fsl.push k 34 + assert_equal {34 33} [$rd read] + } +} From 61d46950dcd15741db5599b296a01c5882531c4f Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Mon, 4 Nov 2019 16:36:06 +0100 Subject: [PATCH 249/320] fix unreported overflow in autogerenared stream IDs --- src/t_stream.c | 23 +++++++++++++---------- tests/unit/type/stream.tcl | 6 ++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index ea9a620f1..58b59f521 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -173,9 +173,19 @@ int streamCompareID(streamID *a, streamID *b) { * C_ERR if an ID was given via 'use_id', but adding it failed since the * current top ID is greater or equal. */ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) { - /* If an ID was given, check that it's greater than the last entry ID - * or return an error. */ - if (use_id && streamCompareID(use_id,&s->last_id) <= 0) return C_ERR; + + /* Generate the new entry ID. */ + streamID id; + if (use_id) + id = *use_id; + else + streamNextID(&s->last_id,&id); + + /* Check that the new ID is greater than the last entry ID + * or return an error. Automatically generated IDs might + * overflow (and wrap-around) when incrementing the sequence + part. */ + if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR; /* Add the new entry. */ raxIterator ri; @@ -192,13 +202,6 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ } raxStop(&ri); - /* Generate the new entry ID. */ - streamID id; - if (use_id) - id = *use_id; - else - streamNextID(&s->last_id,&id); - /* We have to add the key into the radix tree in lexicographic order, * to do so we consider the ID as a single 128 bit number written in * big endian, so that the most significant bytes are the first ones. */ diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index a7415ae8d..aa9c5f3a9 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -79,6 +79,12 @@ start_server { assert {[streamCompareID $id2 $id3] == -1} } + test {XADD IDs correctly report an error when overflowing} { + r DEL mystream + r xadd mystream 18446744073709551615-18446744073709551615 a b + assert_error ERR* {r xadd mystream * c d} + } + test {XADD with MAXLEN option} { r DEL mystream for {set j 0} {$j < 1000} {incr j} { From 9822d541d475e4181ee7f757e22a9c3fe931026a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 4 Nov 2019 19:30:31 +0200 Subject: [PATCH 250/320] Test coverage for new module APIs: dbsize, flushall, randomkey, lru get/set --- tests/modules/misc.c | 69 +++++++++++++++++++++++++++++++++++ tests/unit/moduleapi/misc.tcl | 19 ++++++++++ 2 files changed, 88 insertions(+) diff --git a/tests/modules/misc.c b/tests/modules/misc.c index fd892f52c..7701a9c7c 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -40,6 +40,65 @@ int test_call_info(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } +int test_flushall(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModule_ResetDataset(1, 0); + RedisModule_ReplyWithCString(ctx, "Ok"); + return REDISMODULE_OK; +} + +int test_dbsize(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + long long ll = RedisModule_DbSize(ctx); + RedisModule_ReplyWithLongLong(ctx, ll); + return REDISMODULE_OK; +} + +int test_randomkey(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + RedisModuleString *str = RedisModule_RandomKey(ctx); + RedisModule_ReplyWithString(ctx, str); + RedisModule_FreeString(ctx, str); + return REDISMODULE_OK; +} + +int test_getlru(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc<2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModuleString *keyname = argv[1]; + RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); + long long lru, lfu; + RedisModule_GetLRUOrLFU(key, &lfu, &lru); + RedisModule_ReplyWithLongLong(ctx, lru); + RedisModule_CloseKey(key); + return REDISMODULE_OK; +} + +int test_setlru(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc<3) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModuleString *keyname = argv[1]; + RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, REDISMODULE_WRITE|REDISMODULE_OPEN_KEY_NOTOUCH); + long long lru; + RedisModule_StringToLongLong(argv[2], &lru); + RedisModule_SetLRUOrLFU(key, -1, lru); + RedisModule_ReplyWithCString(ctx, "Ok"); + RedisModule_CloseKey(key); + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -50,6 +109,16 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"test.call_info", test_call_info,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.flushall", test_flushall,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.dbsize", test_dbsize,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.randomkey", test_randomkey,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.setlru", test_setlru,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.getlru", test_getlru,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl index d392aeab0..ebfa9631f 100644 --- a/tests/unit/moduleapi/misc.tcl +++ b/tests/unit/moduleapi/misc.tcl @@ -16,4 +16,23 @@ start_server {tags {"modules"}} { assert { [string match "*cmdstat_module*" $info] } } + test {test module db commands} { + r set x foo + set key [r test.randomkey] + assert_equal $key "x" + assert_equal [r test.dbsize] 1 + r test.flushall + assert_equal [r test.dbsize] 0 + } + + test {test modle lru api} { + r set x foo + set lru [r test.getlru x] + assert { $lru <= 1 } + r test.setlru x 100 + set idle [r object idletime x] + assert { $idle >= 100 } + set lru [r test.getlru x] + assert { $lru >= 100 } + } } From a24f1971f418189ed36e577d1cbf5976216ccf3d Mon Sep 17 00:00:00 2001 From: artix Date: Thu, 31 Oct 2019 16:51:30 +0100 Subject: [PATCH 251/320] Start support for long double in modules New API: - RedisModule_StringToLongDouble - RedisModule_CreateStringFromLongDouble - RedisModule_ReplyWithLongDouble --- src/module.c | 40 ++++++++++++++++++++++++++++++++++++++++ src/redismodule.h | 6 ++++++ 2 files changed, 46 insertions(+) diff --git a/src/module.c b/src/module.c index ad34e7b64..e78ac0c57 100644 --- a/src/module.c +++ b/src/module.c @@ -1012,6 +1012,20 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll return RM_CreateString(ctx,buf,len); } +/* Like RedisModule_CreatString(), but creates a string starting from a long + * double. + * + * The returned string must be released with RedisModule_FreeString() or by + * enabling automatic memory management. + * + * The passed context 'ctx' may be NULL if necessary, see the + * RedisModule_CreateString() documentation for more info. */ +RedisModuleString *RM_CreateStringFromLongDouble(RedisModuleCtx *ctx, long double ld, int humanfriendly) { + char buf[MAX_LONG_DOUBLE_CHARS]; + size_t len = ld2string(buf,sizeof(buf),ld,humanfriendly); + return RM_CreateString(ctx,buf,len); +} + /* Like RedisModule_CreatString(), but creates a string starting from another * RedisModuleString. * @@ -1116,6 +1130,14 @@ int RM_StringToDouble(const RedisModuleString *str, double *d) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +/* Convert the string into a long double, storing it at `*ld`. + * Returns REDISMODULE_OK on success or REDISMODULE_ERR if the string is + * not a valid string representation of a double value. */ +int RM_StringToLongDouble(const RedisModuleString *str, long double *ld) { + int retval = string2ld(str->ptr,sdslen(str->ptr),ld); + return retval ? REDISMODULE_OK : REDISMODULE_ERR; +} + /* Compare two string objects, returning -1, 0 or 1 respectively if * a < b, a == b, a > b. Strings are compared byte by byte as two * binary blobs without any encoding care / collation attempt. */ @@ -1442,6 +1464,21 @@ int RM_ReplyWithDouble(RedisModuleCtx *ctx, double d) { return REDISMODULE_OK; } +/* Send a string reply obtained converting the long double 'ld' into a bulk + * string. This function is basically equivalent to converting a long double + * into a string into a C buffer, and then calling the function + * RedisModule_ReplyWithStringBuffer() with the buffer and length. + * The double string uses human readable formatting (see + * `addReplyHumanLongDouble` in networking.c). + * + * The function always returns REDISMODULE_OK. */ +int RM_ReplyWithLongDouble(RedisModuleCtx *ctx, long double ld) { + client *c = moduleGetReplyClient(ctx); + if (c == NULL) return REDISMODULE_OK; + addReplyHumanLongDouble(c, ld); + return REDISMODULE_OK; +} + /* -------------------------------------------------------------------------- * Commands replication API * -------------------------------------------------------------------------- */ @@ -6797,6 +6834,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ReplyWithNull); REGISTER_API(ReplyWithCallReply); REGISTER_API(ReplyWithDouble); + REGISTER_API(ReplyWithLongDouble); REGISTER_API(GetSelectedDb); REGISTER_API(SelectDb); REGISTER_API(OpenKey); @@ -6807,6 +6845,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(ListPop); REGISTER_API(StringToLongLong); REGISTER_API(StringToDouble); + REGISTER_API(StringToLongDouble); REGISTER_API(Call); REGISTER_API(CallReplyProto); REGISTER_API(FreeCallReply); @@ -6818,6 +6857,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(CreateStringFromCallReply); REGISTER_API(CreateString); REGISTER_API(CreateStringFromLongLong); + REGISTER_API(CreateStringFromLongDouble); REGISTER_API(CreateStringFromString); REGISTER_API(CreateStringPrintf); REGISTER_API(FreeString); diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..d390263b2 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -458,6 +458,7 @@ size_t REDISMODULE_API_FUNC(RedisModule_CallReplyLength)(RedisModuleCallReply *r RedisModuleCallReply *REDISMODULE_API_FUNC(RedisModule_CallReplyArrayElement)(RedisModuleCallReply *reply, size_t idx); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateString)(RedisModuleCtx *ctx, const char *ptr, size_t len); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongLong)(RedisModuleCtx *ctx, long long ll); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromLongDouble)(RedisModuleCtx *ctx, long double ld, int humanfriendly); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringFromString)(RedisModuleCtx *ctx, const RedisModuleString *str); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_CreateStringPrintf)(RedisModuleCtx *ctx, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_FreeString)(RedisModuleCtx *ctx, RedisModuleString *str); @@ -475,9 +476,11 @@ int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyString)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ReplyWithVerbatimString)(RedisModuleCtx *ctx, const char *buf, size_t len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongDouble)(RedisModuleCtx *ctx, double d); int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply); int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll); int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(const RedisModuleString *str, double *d); +int REDISMODULE_API_FUNC(RedisModule_StringToLongDouble)(const RedisModuleString *str, long double *d); void REDISMODULE_API_FUNC(RedisModule_AutoMemory)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_Replicate)(RedisModuleCtx *ctx, const char *cmdname, const char *fmt, ...); int REDISMODULE_API_FUNC(RedisModule_ReplicateVerbatim)(RedisModuleCtx *ctx); @@ -666,6 +669,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ReplyWithNull); REDISMODULE_GET_API(ReplyWithCallReply); REDISMODULE_GET_API(ReplyWithDouble); + REDISMODULE_GET_API(ReplyWithLongDouble); REDISMODULE_GET_API(GetSelectedDb); REDISMODULE_GET_API(SelectDb); REDISMODULE_GET_API(OpenKey); @@ -676,6 +680,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(ListPop); REDISMODULE_GET_API(StringToLongLong); REDISMODULE_GET_API(StringToDouble); + REDISMODULE_GET_API(StringToLongDouble); REDISMODULE_GET_API(Call); REDISMODULE_GET_API(CallReplyProto); REDISMODULE_GET_API(FreeCallReply); @@ -687,6 +692,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(CreateStringFromCallReply); REDISMODULE_GET_API(CreateString); REDISMODULE_GET_API(CreateStringFromLongLong); + REDISMODULE_GET_API(CreateStringFromLongDouble); REDISMODULE_GET_API(CreateStringFromString); REDISMODULE_GET_API(CreateStringPrintf); REDISMODULE_GET_API(FreeString); From 52a00f773b0cf56ad0cc4e28ad0d40f15b24e28f Mon Sep 17 00:00:00 2001 From: artix Date: Mon, 4 Nov 2019 18:04:35 +0100 Subject: [PATCH 252/320] Fix RedisModule_ReplyWithLongDouble ptr definition, add tests --- src/redismodule.h | 2 +- tests/modules/misc.c | 40 ++++++++++++++++++++++++++++++++++- tests/unit/moduleapi/misc.tcl | 5 +++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/redismodule.h b/src/redismodule.h index d390263b2..98939e93c 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -476,7 +476,7 @@ int REDISMODULE_API_FUNC(RedisModule_ReplyWithEmptyString)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ReplyWithVerbatimString)(RedisModuleCtx *ctx, const char *buf, size_t len); int REDISMODULE_API_FUNC(RedisModule_ReplyWithNull)(RedisModuleCtx *ctx); int REDISMODULE_API_FUNC(RedisModule_ReplyWithDouble)(RedisModuleCtx *ctx, double d); -int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongDouble)(RedisModuleCtx *ctx, double d); +int REDISMODULE_API_FUNC(RedisModule_ReplyWithLongDouble)(RedisModuleCtx *ctx, long double d); int REDISMODULE_API_FUNC(RedisModule_ReplyWithCallReply)(RedisModuleCtx *ctx, RedisModuleCallReply *reply); int REDISMODULE_API_FUNC(RedisModule_StringToLongLong)(const RedisModuleString *str, long long *ll); int REDISMODULE_API_FUNC(RedisModule_StringToDouble)(const RedisModuleString *str, double *d); diff --git a/tests/modules/misc.c b/tests/modules/misc.c index fd892f52c..52a4c4d41 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -40,6 +40,43 @@ int test_call_info(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } +int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + long double ld = 0.00000000000000001L; + const char *ldstr = "0.00000000000000001"; + RedisModuleString *s1 = RedisModule_CreateStringFromLongDouble(ctx, ld, 1); + RedisModuleString *s2 = + RedisModule_CreateString(ctx, ldstr, strlen(ldstr)); + if (RedisModule_StringCompare(s1, s2) != 0) { + char err[4096]; + snprintf(err, 4096, + "Failed to convert long double to string ('%s' != '%s')", + RedisModule_StringPtrLen(s1, NULL), + RedisModule_StringPtrLen(s2, NULL)); + RedisModule_ReplyWithError(ctx, err); + goto final; + } + long double ld2 = 0; + if (RedisModule_StringToLongDouble(s2, &ld2) == REDISMODULE_ERR) { + RedisModule_ReplyWithError(ctx, + "Failed to convert string to long double"); + goto final; + } + if (ld2 != ld) { + char err[4096]; + snprintf(err, 4096, + "Failed to convert string to long double (%.40Lf != %.40Lf)", + ld2, + ld); + RedisModule_ReplyWithError(ctx, err); + goto final; + } + RedisModule_ReplyWithLongDouble(ctx, ld2); +final: + RedisModule_FreeString(ctx, s1); + RedisModule_FreeString(ctx, s2); + return REDISMODULE_OK; +} + int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -50,6 +87,7 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"test.call_info", test_call_info,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; - + if (RedisModule_CreateCommand(ctx,"test.ld_conversion", test_ld_conv, "",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl index d392aeab0..21529747b 100644 --- a/tests/unit/moduleapi/misc.tcl +++ b/tests/unit/moduleapi/misc.tcl @@ -16,4 +16,9 @@ start_server {tags {"modules"}} { assert { [string match "*cmdstat_module*" $info] } } + test {test long double conversions} { + set ld [r test.ld_conversion] + assert {[string match $ld "0.00000000000000001"]} + } + } From 29d1e22cd1b7ce53459bed97f1fa157d0c497468 Mon Sep 17 00:00:00 2001 From: artix Date: Mon, 4 Nov 2019 18:12:03 +0100 Subject: [PATCH 253/320] RM_CreateStringFromLongDouble: use new ld2string 'mode' type --- src/module.c | 3 ++- tests/modules/misc.c | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index e78ac0c57..0ee38a5eb 100644 --- a/src/module.c +++ b/src/module.c @@ -1022,7 +1022,8 @@ RedisModuleString *RM_CreateStringFromLongLong(RedisModuleCtx *ctx, long long ll * RedisModule_CreateString() documentation for more info. */ RedisModuleString *RM_CreateStringFromLongDouble(RedisModuleCtx *ctx, long double ld, int humanfriendly) { char buf[MAX_LONG_DOUBLE_CHARS]; - size_t len = ld2string(buf,sizeof(buf),ld,humanfriendly); + size_t len = ld2string(buf,sizeof(buf),ld, + (humanfriendly ? LD_STR_HUMAN : LD_STR_AUTO)); return RM_CreateString(ctx,buf,len); } diff --git a/tests/modules/misc.c b/tests/modules/misc.c index 52a4c4d41..ba0710538 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -6,6 +6,8 @@ #include #include +#define UNUSED(x) (void)(x) + int test_call_generic(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (argc<2) { @@ -41,6 +43,8 @@ int test_call_info(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) } int test_ld_conv(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); long double ld = 0.00000000000000001L; const char *ldstr = "0.00000000000000001"; RedisModuleString *s1 = RedisModule_CreateStringFromLongDouble(ctx, ld, 1); From a73881f8187b493b4894ebdbc0994f4bf38e673a Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Mon, 11 Jul 2016 16:47:37 +0300 Subject: [PATCH 254/320] Add ModuleDataType to/from string serialization. Add two new functions that leverage the RedisModuleDataType mechanism for RDB serialization/deserialization and make it possible to use it to/from arbitrary strings: * RM_SaveDataTypeToString() * RM_LoadDataTypeFromString() --- runtest-moduleapi | 1 + src/module.c | 55 ++++++++++ src/redismodule.h | 4 + tests/modules/Makefile | 3 +- tests/modules/datatype.c | 161 ++++++++++++++++++++++++++++++ tests/unit/moduleapi/datatype.tcl | 27 +++++ 6 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 tests/modules/datatype.c create mode 100644 tests/unit/moduleapi/datatype.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index e48535126..e5a5a1200 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -22,4 +22,5 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ --single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/datatype "${@}" diff --git a/src/module.c b/src/module.c index ad34e7b64..ca9fb5804 100644 --- a/src/module.c +++ b/src/module.c @@ -3884,6 +3884,59 @@ void RM_DigestEndSequence(RedisModuleDigest *md) { memset(md->o,0,sizeof(md->o)); } +/* Decode a serialized representation of a module data type 'mt' from string + * 'str' and return a newly allocated value, or NULL if decoding failed. + * + * This call basically reuses the 'rdb_load' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + * + * Modules should generally use the REDISMODULE_OPTIONS_HANDLE_IO_ERRORS flag and + * make sure the de-serialization code properly checks and handles IO errors + * (freeing allocated buffers and returning a NULL). + * + * If this is NOT done, Redis will handle corrupted (or just truncated) serialized + * data by producing an error message and terminating the process. + */ + +void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) { + rio payload; + RedisModuleIO io; + + rioInitWithBuffer(&payload, str->ptr); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + + /* All RM_Save*() calls always write a version 2 compatible format, so we + * need to make sure we read the same. + */ + io.ver = 2; + return mt->rdb_load(&io,0); +} + +/* Encode a module data type 'mt' value 'data' into serialized form, and return it + * as a newly allocated RedisModuleString. + * + * This call basically reuses the 'rdb_save' callback which module data types + * implement in order to allow a module to arbitrarily serialize/de-serialize + * keys, similar to how the Redis 'DUMP' and 'RESTORE' commands are implemented. + */ + +RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, const moduleType *mt) { + rio payload; + RedisModuleIO io; + + rioInitWithBuffer(&payload,sdsempty()); + moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); + mt->rdb_save(&io,data); + if (io.error) { + return NULL; + } else { + robj *str = createObject(OBJ_STRING,payload.io.buffer.ptr); + autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str); + return str; + } +} + /* -------------------------------------------------------------------------- * AOF API for modules data types * -------------------------------------------------------------------------- */ @@ -6876,6 +6929,8 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(LoadFloat); REGISTER_API(SaveLongDouble); REGISTER_API(LoadLongDouble); + REGISTER_API(SaveDataTypeToString); + REGISTER_API(LoadDataTypeFromString); REGISTER_API(EmitAOF); REGISTER_API(Log); REGISTER_API(LogIOError); diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..21f3988cf 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -537,6 +537,8 @@ void REDISMODULE_API_FUNC(RedisModule_SaveFloat)(RedisModuleIO *io, float value) float REDISMODULE_API_FUNC(RedisModule_LoadFloat)(RedisModuleIO *io); void REDISMODULE_API_FUNC(RedisModule_SaveLongDouble)(RedisModuleIO *io, long double value); long double REDISMODULE_API_FUNC(RedisModule_LoadLongDouble)(RedisModuleIO *io); +void *REDISMODULE_API_FUNC(RedisModule_LoadDataTypeFromString)(const RedisModuleString *str, const RedisModuleType *mt); +RedisModuleString *REDISMODULE_API_FUNC(RedisModule_SaveDataTypeToString)(RedisModuleCtx *ctx, void *data, const RedisModuleType *mt); void REDISMODULE_API_FUNC(RedisModule_Log)(RedisModuleCtx *ctx, const char *level, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule_LogIOError)(RedisModuleIO *io, const char *levelstr, const char *fmt, ...); void REDISMODULE_API_FUNC(RedisModule__Assert)(const char *estr, const char *file, int line); @@ -745,6 +747,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(LoadFloat); REDISMODULE_GET_API(SaveLongDouble); REDISMODULE_GET_API(LoadLongDouble); + REDISMODULE_GET_API(SaveDataTypeToString); + REDISMODULE_GET_API(LoadDataTypeFromString); REDISMODULE_GET_API(EmitAOF); REDISMODULE_GET_API(Log); REDISMODULE_GET_API(LogIOError); diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 9e27758a2..26627c30c 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -19,7 +19,8 @@ TEST_MODULES = \ propagate.so \ misc.so \ hooks.so \ - blockonkeys.so + blockonkeys.so \ + datatype.so .PHONY: all diff --git a/tests/modules/datatype.c b/tests/modules/datatype.c new file mode 100644 index 000000000..7c39ab457 --- /dev/null +++ b/tests/modules/datatype.c @@ -0,0 +1,161 @@ +/* This module current tests a small subset but should be extended in the future + * for general ModuleDataType coverage. + */ + +#include "redismodule.h" + +static RedisModuleType *datatype = NULL; + +typedef struct { + long long intval; + RedisModuleString *strval; +} DataType; + +static void *datatype_load(RedisModuleIO *io, int encver) { + (void) encver; + + int intval = RedisModule_LoadSigned(io); + if (RedisModule_IsIOError(io)) return NULL; + + RedisModuleString *strval = RedisModule_LoadString(io); + if (RedisModule_IsIOError(io)) return NULL; + + DataType *dt = (DataType *) RedisModule_Alloc(sizeof(DataType)); + dt->intval = intval; + dt->strval = strval; + return dt; +} + +static void datatype_save(RedisModuleIO *io, void *value) { + DataType *dt = (DataType *) value; + RedisModule_SaveSigned(io, dt->intval); + RedisModule_SaveString(io, dt->strval); +} + +static void datatype_free(void *value) { + if (value) { + DataType *dt = (DataType *) value; + + if (dt->strval) RedisModule_FreeString(NULL, dt->strval); + RedisModule_Free(dt); + } +} + +static int datatype_set(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 4) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + long long intval; + + if (RedisModule_StringToLongLong(argv[2], &intval) != REDISMODULE_OK) { + RedisModule_ReplyWithError(ctx, "Invalid integr value"); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + DataType *dt = RedisModule_Calloc(sizeof(DataType), 1); + dt->intval = intval; + dt->strval = argv[3]; + RedisModule_RetainString(ctx, dt->strval); + + RedisModule_ModuleTypeSetValue(key, datatype, dt); + RedisModule_CloseKey(key); + RedisModule_ReplyWithSimpleString(ctx, "OK"); + + return REDISMODULE_OK; +} + +static int datatype_restore(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 3) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + DataType *dt = RedisModule_LoadDataTypeFromString(argv[2], datatype); + if (!dt) { + RedisModule_ReplyWithError(ctx, "Invalid data"); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_WRITE); + RedisModule_ModuleTypeSetValue(key, datatype, dt); + RedisModule_CloseKey(key); + RedisModule_ReplyWithSimpleString(ctx, "OK"); + + return REDISMODULE_OK; +} + +static int datatype_get(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ); + DataType *dt = RedisModule_ModuleTypeGetValue(key); + RedisModule_CloseKey(key); + + RedisModule_ReplyWithArray(ctx, 2); + RedisModule_ReplyWithLongLong(ctx, dt->intval); + RedisModule_ReplyWithString(ctx, dt->strval); + return REDISMODULE_OK; +} + +static int datatype_dump(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ); + DataType *dt = RedisModule_ModuleTypeGetValue(key); + RedisModule_CloseKey(key); + + RedisModuleString *reply = RedisModule_SaveDataTypeToString(ctx, dt, datatype); + if (!reply) { + RedisModule_ReplyWithError(ctx, "Failed to save"); + return REDISMODULE_OK; + } + + RedisModule_ReplyWithString(ctx, reply); + RedisModule_FreeString(ctx, reply); + return REDISMODULE_OK; +} + + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + if (RedisModule_Init(ctx,"datatype",1,REDISMODULE_APIVER_1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + RedisModule_SetModuleOptions(ctx, REDISMODULE_OPTIONS_HANDLE_IO_ERRORS); + + RedisModuleTypeMethods datatype_methods = { + .version = REDISMODULE_TYPE_METHOD_VERSION, + .rdb_load = datatype_load, + .rdb_save = datatype_save, + .free = datatype_free, + }; + + datatype = RedisModule_CreateDataType(ctx, "test___dt", 1, &datatype_methods); + if (datatype == NULL) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"datatype.set", datatype_set,"deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"datatype.get", datatype_get,"",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"datatype.restore", datatype_restore,"deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"datatype.dump", datatype_dump,"",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} diff --git a/tests/unit/moduleapi/datatype.tcl b/tests/unit/moduleapi/datatype.tcl new file mode 100644 index 000000000..c1da696d3 --- /dev/null +++ b/tests/unit/moduleapi/datatype.tcl @@ -0,0 +1,27 @@ +set testmodule [file normalize tests/modules/datatype.so] + +start_server {tags {"modules"}} { + r module load $testmodule + + test {DataType: Test module is sane, GET/SET work.} { + r datatype.set dtkey 100 stringval + assert {[r datatype.get dtkey] eq {100 stringval}} + } + + test {DataType: RM_SaveDataTypeToString(), RM_LoadDataTypeFromString() work} { + r datatype.set dtkey -1111 MyString + set encoded [r datatype.dump dtkey] + + r datatype.restore dtkeycopy $encoded + assert {[r datatype.get dtkeycopy] eq {-1111 MyString}} + } + + test {DataType: Handle truncated RM_LoadDataTypeFromString()} { + r datatype.set dtkey -1111 MyString + set encoded [r datatype.dump dtkey] + set truncated [string range $encoded 0 end-1] + + catch {r datatype.restore dtkeycopy $truncated} e + set e + } {*Invalid*} +} From 576fb9c9d7849b85030210001e7258e258353a69 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 4 Nov 2019 20:32:19 +0800 Subject: [PATCH 255/320] expires: refactoring judgment about whether a key is expired Calling lookupKey*() many times to search a key in one command may get different result. That's because lookupKey*() calls expireIfNeeded(), and delete the key when reach the expire time. So we can get an robj before the expire time, but a NULL after the expire time. The worst is that may lead to Redis crash, for example `RPOPLPUSH foo foo` the first time we get a list form `foo` and hold the pointer, but when we get `foo` again it's expired and deleted. Now we hold a freed memory, when execute rpoplpushHandlePush() redis crash. To fix it, we can refactor the judgment about whether a key is expired, using the same basetime `server.cmd_start_mstime` instead of calling mstime() everytime. --- src/db.c | 2 +- src/server.c | 1 + src/server.h | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 2c0a0cdd3..1a272faae 100644 --- a/src/db.c +++ b/src/db.c @@ -1199,7 +1199,7 @@ int keyIsExpired(redisDb *db, robj *key) { * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ - mstime_t now = server.lua_caller ? server.lua_time_start : mstime(); + mstime_t now = server.lua_caller ? server.lua_time_start : server.cmd_start_mstime; return now > when; } diff --git a/src/server.c b/src/server.c index 8f165113d..99438ccac 100644 --- a/src/server.c +++ b/src/server.c @@ -3596,6 +3596,7 @@ int processCommand(client *c) { queueMultiCommand(c); addReply(c,shared.queued); } else { + server.cmd_start_mstime = mstime(); call(c,CMD_CALL_FULL); c->woff = server.master_repl_offset; if (listLength(server.ready_keys)) diff --git a/src/server.h b/src/server.h index f724f7d64..08eb2eef9 100644 --- a/src/server.h +++ b/src/server.h @@ -1401,6 +1401,7 @@ struct redisServer { time_t timezone; /* Cached timezone. As set by tzset(). */ int daylight_active; /* Currently in daylight saving time. */ long long mstime; /* 'unixtime' with milliseconds resolution. */ + mstime_t cmd_start_mstime; /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ From f75cbab6890676c50e5c68c07b696b728c48c245 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2019 10:14:34 +0100 Subject: [PATCH 256/320] Update PR #6537 patch to for generality. After the thread in #6537 and thanks to the suggestions received, this commit updates the original patch in order to: 1. Solve the problem of updating the time in multiple places by updating it in call(). 2. Avoid introducing a new field but use our cached time. This required some minor refactoring to the function updating the time, and the introduction of a new cached time in microseconds in order to use less gettimeofday() calls. --- src/db.c | 8 ++++++-- src/rdb.c | 2 +- src/server.c | 36 +++++++++++++++++++++++------------- src/server.h | 7 ++++--- 4 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/db.c b/src/db.c index 1a272faae..6c0b190a8 100644 --- a/src/db.c +++ b/src/db.c @@ -1198,8 +1198,12 @@ int keyIsExpired(redisDb *db, robj *key) { * blocked to when the Lua script started. This way a key can expire * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. - * See issue #1525 on Github for more information. */ - mstime_t now = server.lua_caller ? server.lua_time_start : server.cmd_start_mstime; + * See issue #1525 on Github for more information. + * + * Outside the Lua script execution, we use the cached time server.mstime + * that is updated before commands executions in call(). */ + mstime_t now = server.lua_caller ? server.lua_time_start : + server.mstime; return now > when; } diff --git a/src/rdb.c b/src/rdb.c index f530219a4..a499362d5 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1961,7 +1961,7 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { /* The DB can take some non trivial amount of time to load. Update * our cached time since it is used to create and update the last * interaction time with clients and for other important things. */ - updateCachedTime(); + updateCachedTime(0); if (server.masterhost && server.repl_state == REPL_STATE_TRANSFER) replicationSendNewlineToMaster(); loadingProgress(r->processed_bytes); diff --git a/src/server.c b/src/server.c index 99438ccac..2e5d8193b 100644 --- a/src/server.c +++ b/src/server.c @@ -1736,20 +1736,29 @@ void databasesCron(void) { /* We take a cached value of the unix time in the global state because with * virtual memory and aging there is to store the current time in objects at * every object access, and accuracy is not needed. To access a global var is - * a lot faster than calling time(NULL) */ -void updateCachedTime(void) { - server.unixtime = time(NULL); - server.mstime = mstime(); + * a lot faster than calling time(NULL). + * + * This function should be fast because it is called at every command execution + * in call(), so it is possible to decide if to update the daylight saving + * info or not using the 'update_daylight_info' argument. Normally we update + * such info only when calling this function from serverCron() but not when + * calling it from call(). */ +void updateCachedTime(int update_daylight_info) { + server.ustime = ustime(); + server.mstime = server.ustime / 1000; + server.unixtime = server.mstime / 1000; /* To get information about daylight saving time, we need to call * localtime_r and cache the result. However calling localtime_r in this * context is safe since we will never fork() while here, in the main * thread. The logging function will call a thread safe version of * localtime that has no locks. */ - struct tm tm; - time_t ut = server.unixtime; - localtime_r(&ut,&tm); - server.daylight_active = tm.tm_isdst; + if (update_daylight_info) { + struct tm tm; + time_t ut = server.unixtime; + localtime_r(&ut,&tm); + server.daylight_active = tm.tm_isdst; + } } void checkChildrenDone(void) { @@ -1838,7 +1847,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (server.watchdog_period) watchdogScheduleSignal(server.watchdog_period); /* Update the time cache. */ - updateCachedTime(); + updateCachedTime(1); server.hz = server.config_hz; /* Adapt the server.hz value to the number of configured clients. If we have @@ -2252,7 +2261,7 @@ void createSharedObjects(void) { void initServerConfig(void) { int j; - updateCachedTime(); + updateCachedTime(1); getRandomHexChars(server.runid,CONFIG_RUN_ID_SIZE); server.runid[CONFIG_RUN_ID_SIZE] = '\0'; changeReplicationId(); @@ -3238,7 +3247,8 @@ void preventCommandReplication(client *c) { * */ void call(client *c, int flags) { - long long dirty, start, duration; + long long dirty; + ustime_t start, duration; int client_old_flags = c->flags; struct redisCommand *real_cmd = c->cmd; @@ -3259,7 +3269,8 @@ void call(client *c, int flags) { /* Call the command. */ dirty = server.dirty; - start = ustime(); + updateCachedTime(0); + start = server.ustime; c->cmd->proc(c); duration = ustime()-start; dirty = server.dirty-dirty; @@ -3596,7 +3607,6 @@ int processCommand(client *c) { queueMultiCommand(c); addReply(c,shared.queued); } else { - server.cmd_start_mstime = mstime(); call(c,CMD_CALL_FULL); c->woff = server.master_repl_offset; if (listLength(server.ready_keys)) diff --git a/src/server.h b/src/server.h index 08eb2eef9..24dbc079c 100644 --- a/src/server.h +++ b/src/server.h @@ -50,6 +50,7 @@ #include typedef long long mstime_t; /* millisecond time type. */ +typedef long long ustime_t; /* microsecond time type. */ #include "ae.h" /* Event driven programming library */ #include "sds.h" /* Dynamic safe strings */ @@ -1400,8 +1401,8 @@ struct redisServer { _Atomic time_t unixtime; /* Unix time sampled every cron cycle. */ time_t timezone; /* Cached timezone. As set by tzset(). */ int daylight_active; /* Currently in daylight saving time. */ - long long mstime; /* 'unixtime' with milliseconds resolution. */ - mstime_t cmd_start_mstime; + mstime_t mstime; /* 'unixtime' in milliseconds. */ + ustime_t ustime; /* 'unixtime' in microseconds. */ /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ @@ -1997,7 +1998,7 @@ void populateCommandTable(void); void resetCommandTableStats(void); void adjustOpenFilesLimit(void); void closeListeningSockets(int unlink_unix_socket); -void updateCachedTime(void); +void updateCachedTime(int update_daylight_info); void resetServerStats(void); void activeDefragCycle(void); unsigned int getLRUClock(void); From 80dbc9db7b8c671f085681d1ef3abc07fe6dec42 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Nov 2019 09:57:29 +0100 Subject: [PATCH 257/320] Update PR #6537: use a fresh time outside call(). One problem with the solution proposed so far in #6537 is that key lookups outside a command execution via call(), still used a cached time. The cached time needed to be refreshed in multiple places, especially because of modules callbacks from timers, cluster bus, and thread safe contexts, that may use RM_Open(). In order to avoid this problem, this commit introduces the ability to detect if we are inside call(): this way we can use the reference fixed time only when we are in the context of a command execution or Lua script, but for the asynchronous lookups, we can still use mstime() to get a fresh time reference. --- src/db.c | 27 +++++++++++++++++++++------ src/server.c | 4 ++++ src/server.h | 3 ++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/db.c b/src/db.c index 6c0b190a8..5ea3087fc 100644 --- a/src/db.c +++ b/src/db.c @@ -1188,6 +1188,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) { /* Check if the key is expired. */ int keyIsExpired(redisDb *db, robj *key) { mstime_t when = getExpire(db,key); + mstime_t now; if (when < 0) return 0; /* No expire for this key */ @@ -1198,13 +1199,27 @@ int keyIsExpired(redisDb *db, robj *key) { * blocked to when the Lua script started. This way a key can expire * only the first time it is accessed and not in the middle of the * script execution, making propagation to slaves / AOF consistent. - * See issue #1525 on Github for more information. - * - * Outside the Lua script execution, we use the cached time server.mstime - * that is updated before commands executions in call(). */ - mstime_t now = server.lua_caller ? server.lua_time_start : - server.mstime; + * See issue #1525 on Github for more information. */ + if (server.lua_caller) { + now = server.lua_time_start; + } + /* If we are in the middle of a command execution, we still want to use + * a reference time that does not change: in that case we just use the + * cached time, that we update before each call in the call() function. + * This way we avoid that commands such as RPOPLPUSH or similar, that + * may re-open the same key multiple times, can invalidate an already + * open object in a next call, if the next call will see the key expired, + * while the first did not. */ + else if (server.call_depth > 0) { + now = server.mstime; + } + /* For the other cases, we want to use the most fresh time we have. */ + else { + now = mstime(); + } + /* The key expired if the current (virtual or real) time is greater + * than the expire time of the key. */ return now > when; } diff --git a/src/server.c b/src/server.c index 2e5d8193b..acf8eebbc 100644 --- a/src/server.c +++ b/src/server.c @@ -2780,6 +2780,7 @@ void initServer(void) { server.hz = server.config_hz; server.pid = getpid(); server.current_client = NULL; + server.call_depth = 0; server.clients = listCreate(); server.clients_index = raxNew(); server.clients_to_close = listCreate(); @@ -3252,6 +3253,8 @@ void call(client *c, int flags) { int client_old_flags = c->flags; struct redisCommand *real_cmd = c->cmd; + server.call_depth++; + /* Sent the command to clients in MONITOR mode, only if the commands are * not generated from reading an AOF. */ if (listLength(server.monitors) && @@ -3377,6 +3380,7 @@ void call(client *c, int flags) { trackingRememberKeys(caller); } + server.call_depth--; server.stat_numcommands++; } diff --git a/src/server.h b/src/server.h index 24dbc079c..4a497c47c 100644 --- a/src/server.h +++ b/src/server.h @@ -1134,7 +1134,8 @@ struct redisServer { list *clients_pending_write; /* There is to write or install handler. */ list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ - client *current_client; /* Current client, only used on crash report */ + client *current_client; /* Current client executing the command. */ + long call_depth; /* call() re-entering count. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ mstime_t clients_pause_end_time; /* Time when we undo clients_paused */ From 2c5a3413dc4848190cc8ea315c1fde60067ea5d7 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Wed, 28 Aug 2019 18:08:07 +0300 Subject: [PATCH 258/320] expose used memory via redismodule api The exposed functions: 1. RedisModule_GetUsedMemoryPercentage - return the used memory 2. RedisModue_MallocSize - return for a given pointer, the amount of memory allocated for this pointer --- src/module.c | 22 ++++++++++++++++++++++ src/redismodule.h | 4 ++++ 2 files changed, 26 insertions(+) diff --git a/src/module.c b/src/module.c index ad34e7b64..75c98f12d 100644 --- a/src/module.c +++ b/src/module.c @@ -5891,6 +5891,26 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/* + * For a given pointer, return The amount of memory + * allocated for this pointer. + */ +size_t RM_MallocSize(void* ptr){ + return zmalloc_size(ptr); +} + +/* + * Return the a number between 0 to 1 indicating + * the amount of memory currently used. + * 0 - no memory limit + * 1 and above, memory limit reached. + */ +float RM_GetUsedMemoryPercentage(){ + float level; + getMaxmemoryState(NULL, NULL, NULL, &level); + return level; +} + /* -------------------------------------------------------------------------- * Module fork API * -------------------------------------------------------------------------- */ @@ -6971,4 +6991,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(BlockClientOnKeys); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); + REGISTER_API(GetUsedMemoryPercentage); + REGISTER_API(MallocSize); } diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..259ea2532 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -633,6 +633,8 @@ int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandF int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); +float REDISMODULE_API_FUNC(RedisModule_GetUsedMemoryPercentage)(); +size_t REDISMODULE_API_FUNC(RedisModule_MallocSize)(void* ptr); #endif #define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) @@ -842,6 +844,8 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(Fork); REDISMODULE_GET_API(ExitFromChild); REDISMODULE_GET_API(KillForkChild); + REDISMODULE_GET_API(GetUsedMemoryPercentage); + REDISMODULE_GET_API(MallocSize); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; From 667c9cd8c762760cd22f43583e42dda8ef2c23fa Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Wed, 6 Nov 2019 11:25:21 +0200 Subject: [PATCH 259/320] return value between 0 to 100 instead of 0 to 1. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 75c98f12d..1671934b8 100644 --- a/src/module.c +++ b/src/module.c @@ -5903,12 +5903,12 @@ size_t RM_MallocSize(void* ptr){ * Return the a number between 0 to 1 indicating * the amount of memory currently used. * 0 - no memory limit - * 1 and above, memory limit reached. + * 100 and above, memory limit reached. */ float RM_GetUsedMemoryPercentage(){ float level; getMaxmemoryState(NULL, NULL, NULL, &level); - return level; + return level * 100; } /* -------------------------------------------------------------------------- From 0a6f42fc8ebbdc94e8a75a3514ab19d91f8559b1 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Wed, 6 Nov 2019 12:17:09 +0200 Subject: [PATCH 260/320] changed GetUsedMemoryPresentage -> GetUsedMemoryRatio and return value between 0 and 1. --- src/module.c | 6 +++--- src/redismodule.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/module.c b/src/module.c index 1671934b8..4dbc57ad6 100644 --- a/src/module.c +++ b/src/module.c @@ -5905,10 +5905,10 @@ size_t RM_MallocSize(void* ptr){ * 0 - no memory limit * 100 and above, memory limit reached. */ -float RM_GetUsedMemoryPercentage(){ +float RM_GetUsedMemoryRatio(){ float level; getMaxmemoryState(NULL, NULL, NULL, &level); - return level * 100; + return level; } /* -------------------------------------------------------------------------- @@ -6991,6 +6991,6 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(BlockClientOnKeys); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); - REGISTER_API(GetUsedMemoryPercentage); + REGISTER_API(GetUsedMemoryRatio); REGISTER_API(MallocSize); } diff --git a/src/redismodule.h b/src/redismodule.h index 259ea2532..c09e03541 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -633,7 +633,7 @@ int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandF int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); -float REDISMODULE_API_FUNC(RedisModule_GetUsedMemoryPercentage)(); +float REDISMODULE_API_FUNC(RedisModule_GetUsedMemoryRatio)(); size_t REDISMODULE_API_FUNC(RedisModule_MallocSize)(void* ptr); #endif @@ -844,7 +844,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(Fork); REDISMODULE_GET_API(ExitFromChild); REDISMODULE_GET_API(KillForkChild); - REDISMODULE_GET_API(GetUsedMemoryPercentage); + REDISMODULE_GET_API(GetUsedMemoryRatio); REDISMODULE_GET_API(MallocSize); #endif From ce913eea4c5831ef8deda7662e26aad650c558bf Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 6 Nov 2019 15:48:46 +0530 Subject: [PATCH 261/320] Support streams in general module API functions Fixes GitHub issue #6492 Added stream support in RM_KeyType and RM_ValueLength. Also moduleDelKeyIfEmpty was updated, even though it has no effect now (It will be relevant when stream type direct API will be coded - i.e. RM_StreamAdd) --- src/module.c | 5 ++++- src/redismodule.h | 1 + src/stream.h | 1 + src/t_stream.c | 6 ++++++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index ad34e7b64..6dd15c7db 100644 --- a/src/module.c +++ b/src/module.c @@ -518,7 +518,8 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) { case OBJ_LIST: isempty = listTypeLength(o) == 0; break; case OBJ_SET: isempty = setTypeSize(o) == 0; break; case OBJ_ZSET: isempty = zsetLength(o) == 0; break; - case OBJ_HASH : isempty = hashTypeLength(o) == 0; break; + case OBJ_HASH: isempty = hashTypeLength(o) == 0; break; + case OBJ_STREAM: isempty = streamLength(o) == 0; break; default: isempty = 0; } @@ -1916,6 +1917,7 @@ int RM_KeyType(RedisModuleKey *key) { case OBJ_ZSET: return REDISMODULE_KEYTYPE_ZSET; case OBJ_HASH: return REDISMODULE_KEYTYPE_HASH; case OBJ_MODULE: return REDISMODULE_KEYTYPE_MODULE; + case OBJ_STREAM: return REDISMODULE_KEYTYPE_STREAM; default: return 0; } } @@ -1933,6 +1935,7 @@ size_t RM_ValueLength(RedisModuleKey *key) { case OBJ_SET: return setTypeSize(key->value); case OBJ_ZSET: return zsetLength(key->value); case OBJ_HASH: return hashTypeLength(key->value); + case OBJ_STREAM: return streamLength(key->value); default: return 0; } } diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..40a73454d 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -33,6 +33,7 @@ #define REDISMODULE_KEYTYPE_SET 4 #define REDISMODULE_KEYTYPE_ZSET 5 #define REDISMODULE_KEYTYPE_MODULE 6 +#define REDISMODULE_KEYTYPE_STREAM 7 /* Reply types. */ #define REDISMODULE_REPLY_UNKNOWN -1 diff --git a/src/stream.h b/src/stream.h index 1163b3527..7de769ba1 100644 --- a/src/stream.h +++ b/src/stream.h @@ -98,6 +98,7 @@ struct client; stream *streamNew(void); void freeStream(stream *s); +unsigned long streamLength(const robj *subject); size_t streamReplyWithRange(client *c, stream *s, streamID *start, streamID *end, size_t count, int rev, streamCG *group, streamConsumer *consumer, int flags, streamPropInfo *spi); void streamIteratorStart(streamIterator *si, stream *s, streamID *start, streamID *end, int rev); int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields); diff --git a/src/t_stream.c b/src/t_stream.c index 58b59f521..fc187e318 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -67,6 +67,12 @@ void freeStream(stream *s) { zfree(s); } +/* Return the length of a stream. */ +unsigned long streamLength(const robj *subject) { + stream *s = subject->ptr; + return s->length; +} + /* Generate the next stream item ID given the previous one. If the current * milliseconds Unix time is greater than the previous one, just use this * as time part and start with sequence part of zero. Otherwise we use the From 46ec9642ca15ecc644a4ec449e5394cae0eb1317 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Wed, 6 Nov 2019 12:26:03 +0200 Subject: [PATCH 262/320] fix documentation --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 4dbc57ad6..947cc3083 100644 --- a/src/module.c +++ b/src/module.c @@ -5903,7 +5903,7 @@ size_t RM_MallocSize(void* ptr){ * Return the a number between 0 to 1 indicating * the amount of memory currently used. * 0 - no memory limit - * 100 and above, memory limit reached. + * 1 and above, memory limit reached. */ float RM_GetUsedMemoryRatio(){ float level; From 92e847d054562f2a7b8a3f25aaada2803b6c0ad3 Mon Sep 17 00:00:00 2001 From: Patrick Valsecchi Date: Thu, 7 Nov 2019 08:49:19 +0100 Subject: [PATCH 263/320] Redis sentinel kill pubsub client connections as well When a redis instance becomes a slave, sentinel also kills pubsub clients. Closes #6545 --- src/sentinel.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 0490db4e9..d5f22b97f 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -465,6 +465,12 @@ struct redisCommand sentinelcmds[] = { {"hello",helloCommand,-2,"no-script fast",0,NULL,0,0,0,0,0} }; +/* List of client types that are killed when an instance becomes a slave */ +const char* killedClientTypes[] = { + "normal", + "pubsub" +}; + /* This function overwrites a few normal Redis config default with Sentinel * specific defaults. */ void initSentinelConfig(void) { @@ -3949,6 +3955,7 @@ char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t epoch) { int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { char portstr[32]; int retval; + unsigned int curType; ll2string(portstr,sizeof(portstr),port); @@ -3993,11 +4000,14 @@ int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { * an issue because CLIENT is variadic command, so Redis will not * recognized as a syntax error, and the transaction will not fail (but * only the unsupported command will fail). */ - retval = redisAsyncCommand(ri->link->cc, - sentinelDiscardReplyCallback, ri, "%s KILL TYPE normal", - sentinelInstanceMapCommand(ri,"CLIENT")); - if (retval == C_ERR) return retval; - ri->link->pending_commands++; + for (curType = 0; curType < sizeof(killedClientTypes)/sizeof(killedClientTypes[0]); ++curType) { + retval = redisAsyncCommand(ri->link->cc, + sentinelDiscardReplyCallback, ri, "%s KILL TYPE %s", + sentinelInstanceMapCommand(ri,"CLIENT"), + killedClientTypes[curType]); + if (retval == C_ERR) return retval; + ri->link->pending_commands++; + } retval = redisAsyncCommand(ri->link->cc, sentinelDiscardReplyCallback, ri, "%s", From f35548211f87a1cbf66a2fd2f3196957c330d929 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 8 Nov 2019 19:06:51 +0800 Subject: [PATCH 264/320] expires & blocking: handle ready keys as call() --- src/blocked.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/blocked.c b/src/blocked.c index 14c2ff830..dea4cc57a 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -514,6 +514,9 @@ void handleClientsBlockedOnKeys(void) { * we can safely call signalKeyAsReady() against this key. */ dictDelete(rl->db->ready_keys,rl->key); + server.call_depth++; + updateCachedTime(0); + /* Serve clients blocked on list key. */ robj *o = lookupKeyWrite(rl->db,rl->key); @@ -530,6 +533,8 @@ void handleClientsBlockedOnKeys(void) { serveClientsBlockedOnKeyByModule(rl); } + server.call_depth++; + /* Free this item. */ decrRefCount(rl->key); zfree(rl); From bd206b583a58f89dd1e2743b678d148dad31174b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 10 Nov 2019 09:21:19 +0200 Subject: [PATCH 265/320] fix leak in module api rdb test recently added more reads into that function, if a later read fails, i must either free what's already allocated, or return the pointer so that the free callback will release it. --- tests/modules/testrdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index 8a262e8a7..7c04bb4ef 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -18,8 +18,11 @@ void *testrdb_type_load(RedisModuleIO *rdb, int encver) { RedisModuleString *str = RedisModule_LoadString(rdb); float f = RedisModule_LoadFloat(rdb); long double ld = RedisModule_LoadLongDouble(rdb); - if (RedisModule_IsIOError(rdb)) + if (RedisModule_IsIOError(rdb)) { + RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb); + RedisModule_FreeString(ctx, str); return NULL; + } /* Using the values only after checking for io errors. */ assert(count==1); assert(encver==1); From 86e1f73bd6bc2d959653d1d3c78dc1358349a993 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 10 Nov 2019 09:04:39 +0200 Subject: [PATCH 266/320] rename RN_SetLRUOrLFU -> RM_SetLRU and RN_SetLFU - the API name was odd, separated to two apis one for LRU and one for LFU - the LRU idle time was in 1 second resolution, which might be ok for RDB and RESTORE, but i think modules may need higher resolution - adding tests for LFU and for handling maxmemory policy mismatch --- src/cluster.c | 2 +- src/module.c | 64 +++++++++++++++++++++------------ src/object.c | 4 +-- src/rdb.c | 2 +- src/redismodule.h | 12 ++++--- src/server.h | 2 +- tests/modules/misc.c | 66 +++++++++++++++++++++++++++++------ tests/unit/moduleapi/misc.tcl | 33 ++++++++++++++++-- 8 files changed, 141 insertions(+), 44 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index a7d8a02c3..9e6ddb2c4 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -4966,7 +4966,7 @@ void restoreCommand(client *c) { if (!absttl) ttl+=mstime(); setExpire(c,c->db,c->argv[1],ttl); } - objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock); + objectSetLRUOrLFU(obj,lfu_freq,lru_idle,lru_clock,1000); signalModifiedKey(c->db,c->argv[1]); addReply(c,shared.ok); server.dirty++; diff --git a/src/module.c b/src/module.c index ad34e7b64..cd79a5908 100644 --- a/src/module.c +++ b/src/module.c @@ -6736,35 +6736,53 @@ size_t moduleCount(void) { return dictSize(modules); } -/* Set the key LRU/LFU depending on server.maxmemory_policy. - * The lru_idle arg is idle time in seconds, and is only relevant if the - * eviction policy is LRU based. - * The lfu_freq arg is a logarithmic counter that provides an indication of - * the access frequencyonly (must be <= 255) and is only relevant if the - * eviction policy is LFU based. - * Either or both of them may be <0, in that case, nothing is set. */ -/* return value is an indication if the lru field was updated or not. */ -int RM_SetLRUOrLFU(RedisModuleKey *key, long long lfu_freq, long long lru_idle) { +/* Set the key last access time for LRU based eviction. not relevent if the + * servers's maxmemory policy is LFU based. Value is idle time in milliseconds. + * returns REDISMODULE_OK if the LRU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLRU(RedisModuleKey *key, mstime_t lru_idle) { if (!key->value) return REDISMODULE_ERR; - if (objectSetLRUOrLFU(key->value, lfu_freq, lru_idle, lru_idle>=0 ? LRU_CLOCK() : 0)) + if (objectSetLRUOrLFU(key->value, -1, lru_idle, lru_idle>=0 ? LRU_CLOCK() : 0, 1)) return REDISMODULE_OK; return REDISMODULE_ERR; } -/* Gets the key LRU or LFU (depending on the current eviction policy). - * One will be set to the appropiate return value, and the other will be set to -1. - * see RedisModule_SetLRUOrLFU for units and ranges. - * return value is an indication of success. */ -int RM_GetLRUOrLFU(RedisModuleKey *key, long long *lfu_freq, long long *lru_idle) { - *lru_idle = *lfu_freq = -1; +/* Gets the key last access time. + * Value is idletime in milliseconds or -1 if the server's eviction policy is + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLRU(RedisModuleKey *key, mstime_t *lru_idle) { + *lru_idle = -1; if (!key->value) return REDISMODULE_ERR; - if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) + return REDISMODULE_OK; + *lru_idle = estimateObjectIdleTime(key->value); + return REDISMODULE_OK; +} + +/* Set the key access frequency. only relevant if the server's maxmemory policy + * is LFU based. + * The frequency is a logarithmic counter that provides an indication of + * the access frequencyonly (must be <= 255). + * returns REDISMODULE_OK if the LFU was updated, REDISMODULE_ERR otherwise. */ +int RM_SetLFU(RedisModuleKey *key, long long lfu_freq) { + if (!key->value) + return REDISMODULE_ERR; + if (objectSetLRUOrLFU(key->value, lfu_freq, -1, 0, 1)) + return REDISMODULE_OK; + return REDISMODULE_ERR; +} + +/* Gets the key access frequency or -1 if the server's eviction policy is not + * LFU based. + * returns REDISMODULE_OK if when key is valid. */ +int RM_GetLFU(RedisModuleKey *key, long long *lfu_freq) { + *lfu_freq = -1; + if (!key->value) + return REDISMODULE_ERR; + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) *lfu_freq = LFUDecrAndReturn(key->value); - } else { - *lru_idle = estimateObjectIdleTime(key->value)/1000; - } return REDISMODULE_OK; } @@ -6966,8 +6984,10 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetClientInfoById); REGISTER_API(PublishMessage); REGISTER_API(SubscribeToServerEvent); - REGISTER_API(SetLRUOrLFU); - REGISTER_API(GetLRUOrLFU); + REGISTER_API(SetLRU); + REGISTER_API(GetLRU); + REGISTER_API(SetLFU); + REGISTER_API(GetLFU); REGISTER_API(BlockClientOnKeys); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); diff --git a/src/object.c b/src/object.c index 5e9a99dec..c6d89bfa7 100644 --- a/src/object.c +++ b/src/object.c @@ -1210,7 +1210,7 @@ sds getMemoryDoctorReport(void) { * is MAXMEMORY_FLAG_LRU. * Either or both of them may be <0, in that case, nothing is set. */ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, - long long lru_clock) { + long long lru_clock, int lru_multiplier) { if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { if (lfu_freq >= 0) { serverAssert(lfu_freq <= 255); @@ -1222,7 +1222,7 @@ int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, * according to the LRU clock resolution this Redis * instance was compiled with (normally 1000 ms, so the * below statement will expand to lru_idle*1000/1000. */ - lru_idle = lru_idle*1000/LRU_CLOCK_RESOLUTION; + lru_idle = lru_idle*lru_multiplier/LRU_CLOCK_RESOLUTION; long lru_abs = lru_clock - lru_idle; /* Absolute access time. */ /* If the LRU field underflows (since LRU it is a wrapping * clock), the best we can do is to provide a large enough LRU diff --git a/src/rdb.c b/src/rdb.c index b569edfea..301a33642 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2239,7 +2239,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if (expiretime != -1) setExpire(NULL,db,key,expiretime); /* Set usage information (for eviction). */ - objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock); + objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000); /* Decrement the key refcount since dbAdd() will take its * own reference. */ diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..e388d7439 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -583,8 +583,10 @@ int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldDouble)(RedisModuleInfoCtx *ctx int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldLongLong)(RedisModuleInfoCtx *ctx, char *field, long long value); int REDISMODULE_API_FUNC(RedisModule_InfoAddFieldULongLong)(RedisModuleInfoCtx *ctx, char *field, unsigned long long value); int REDISMODULE_API_FUNC(RedisModule_SubscribeToServerEvent)(RedisModuleCtx *ctx, RedisModuleEvent event, RedisModuleEventCallback callback); -int REDISMODULE_API_FUNC(RedisModule_SetLRUOrLFU)(RedisModuleKey *key, long long lfu_freq, long long lru_idle); -int REDISMODULE_API_FUNC(RedisModule_GetLRUOrLFU)(RedisModuleKey *key, long long *lfu_freq, long long *lru_idle); +int REDISMODULE_API_FUNC(RedisModule_SetLRU)(RedisModuleKey *key, mstime_t lru_idle); +int REDISMODULE_API_FUNC(RedisModule_GetLRU)(RedisModuleKey *key, mstime_t *lru_idle); +int REDISMODULE_API_FUNC(RedisModule_SetLFU)(RedisModuleKey *key, long long lfu_freq); +int REDISMODULE_API_FUNC(RedisModule_GetLFU)(RedisModuleKey *key, long long *lfu_freq); RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientReadyKey)(RedisModuleCtx *ctx); @@ -794,8 +796,10 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(GetClientInfoById); REDISMODULE_GET_API(PublishMessage); REDISMODULE_GET_API(SubscribeToServerEvent); - REDISMODULE_GET_API(SetLRUOrLFU); - REDISMODULE_GET_API(GetLRUOrLFU); + REDISMODULE_GET_API(SetLRU); + REDISMODULE_GET_API(GetLRU); + REDISMODULE_GET_API(SetLFU); + REDISMODULE_GET_API(GetLFU); REDISMODULE_GET_API(BlockClientOnKeys); REDISMODULE_GET_API(SignalKeyAsReady); REDISMODULE_GET_API(GetBlockedClientReadyKey); diff --git a/src/server.h b/src/server.h index 8063dc101..c9ac3003e 100644 --- a/src/server.h +++ b/src/server.h @@ -2089,7 +2089,7 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags); robj *objectCommandLookup(client *c, robj *key); robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, - long long lru_clock); + long long lru_clock, int lru_multiplier); #define LOOKUP_NONE 0 #define LOOKUP_NOTOUCH (1<<0) void dbAdd(redisDb *db, robj *key, robj *val); diff --git a/tests/modules/misc.c b/tests/modules/misc.c index 7701a9c7c..06b5af620 100644 --- a/tests/modules/misc.c +++ b/tests/modules/misc.c @@ -68,16 +68,24 @@ int test_randomkey(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_OK; } +RedisModuleKey *open_key_or_reply(RedisModuleCtx *ctx, RedisModuleString *keyname, int mode) { + RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, mode); + if (!key) { + RedisModule_ReplyWithError(ctx, "key not found"); + return NULL; + } + return key; +} + int test_getlru(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (argc<2) { RedisModule_WrongArity(ctx); return REDISMODULE_OK; } - RedisModuleString *keyname = argv[1]; - RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); - long long lru, lfu; - RedisModule_GetLRUOrLFU(key, &lfu, &lru); + RedisModuleKey *key = open_key_or_reply(ctx, argv[1], REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); + mstime_t lru; + RedisModule_GetLRU(key, &lru); RedisModule_ReplyWithLongLong(ctx, lru); RedisModule_CloseKey(key); return REDISMODULE_OK; @@ -89,12 +97,46 @@ int test_setlru(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) RedisModule_WrongArity(ctx); return REDISMODULE_OK; } - RedisModuleString *keyname = argv[1]; - RedisModuleKey *key = RedisModule_OpenKey(ctx, keyname, REDISMODULE_WRITE|REDISMODULE_OPEN_KEY_NOTOUCH); - long long lru; - RedisModule_StringToLongLong(argv[2], &lru); - RedisModule_SetLRUOrLFU(key, -1, lru); - RedisModule_ReplyWithCString(ctx, "Ok"); + RedisModuleKey *key = open_key_or_reply(ctx, argv[1], REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); + mstime_t lru; + if (RedisModule_StringToLongLong(argv[2], &lru) != REDISMODULE_OK) { + RedisModule_ReplyWithError(ctx, "invalid idle time"); + return REDISMODULE_OK; + } + int was_set = RedisModule_SetLRU(key, lru)==REDISMODULE_OK; + RedisModule_ReplyWithLongLong(ctx, was_set); + RedisModule_CloseKey(key); + return REDISMODULE_OK; +} + +int test_getlfu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc<2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModuleKey *key = open_key_or_reply(ctx, argv[1], REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); + mstime_t lfu; + RedisModule_GetLFU(key, &lfu); + RedisModule_ReplyWithLongLong(ctx, lfu); + RedisModule_CloseKey(key); + return REDISMODULE_OK; +} + +int test_setlfu(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc<3) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + RedisModuleKey *key = open_key_or_reply(ctx, argv[1], REDISMODULE_READ|REDISMODULE_OPEN_KEY_NOTOUCH); + mstime_t lfu; + if (RedisModule_StringToLongLong(argv[2], &lfu) != REDISMODULE_OK) { + RedisModule_ReplyWithError(ctx, "invalid freq"); + return REDISMODULE_OK; + } + int was_set = RedisModule_SetLFU(key, lfu)==REDISMODULE_OK; + RedisModule_ReplyWithLongLong(ctx, was_set); RedisModule_CloseKey(key); return REDISMODULE_OK; } @@ -119,6 +161,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) return REDISMODULE_ERR; if (RedisModule_CreateCommand(ctx,"test.getlru", test_getlru,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.setlfu", test_setlfu,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.getlfu", test_getlfu,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/misc.tcl b/tests/unit/moduleapi/misc.tcl index ebfa9631f..376bc2eed 100644 --- a/tests/unit/moduleapi/misc.tcl +++ b/tests/unit/moduleapi/misc.tcl @@ -26,13 +26,40 @@ start_server {tags {"modules"}} { } test {test modle lru api} { + r config set maxmemory-policy allkeys-lru r set x foo set lru [r test.getlru x] - assert { $lru <= 1 } - r test.setlru x 100 + assert { $lru <= 1000 } + set was_set [r test.setlru x 100000] + assert { $was_set == 1 } set idle [r object idletime x] assert { $idle >= 100 } set lru [r test.getlru x] - assert { $lru >= 100 } + assert { $lru >= 100000 } + r config set maxmemory-policy allkeys-lfu + set lru [r test.getlru x] + assert { $lru == -1 } + set was_set [r test.setlru x 100000] + assert { $was_set == 0 } } + r config set maxmemory-policy allkeys-lru + + test {test modle lfu api} { + r config set maxmemory-policy allkeys-lfu + r set x foo + set lfu [r test.getlfu x] + assert { $lfu >= 1 } + set was_set [r test.setlfu x 100] + assert { $was_set == 1 } + set freq [r object freq x] + assert { $freq <= 100 } + set lfu [r test.getlfu x] + assert { $lfu <= 100 } + r config set maxmemory-policy allkeys-lru + set lfu [r test.getlfu x] + assert { $lfu == -1 } + set was_set [r test.setlfu x 100] + assert { $was_set == 0 } + } + } From f327506889a666913d1d72b17e8acfc1b880bf81 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Mon, 11 Nov 2019 13:06:08 +0000 Subject: [PATCH 267/320] Add missing header --- src/server.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server.h b/src/server.h index a2c94a22a..35fc53c80 100644 --- a/src/server.h +++ b/src/server.h @@ -46,6 +46,7 @@ #include #include #include +#include #include #include From 5642a1da20aeed969f7716d3812e3034a86992eb Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Thu, 17 Oct 2019 15:37:01 +0300 Subject: [PATCH 268/320] Added scan implementation to module api. The implementation expose the following new functions: 1. RedisModule_CursorCreate - allow to create a new cursor object for keys scanning 2. RedisModule_CursorRestart - restart an existing cursor to restart the scan 3. RedisModule_CursorDestroy - destroy an existing cursor 4. RedisModule_Scan - scan keys The RedisModule_Scan function gets a cursor object, a callback and void* (used as user private data). The callback will be called for each key in the database proving the key name and the value as RedisModuleKey. --- runtest-moduleapi | 1 + src/module.c | 133 +++++++++++++++++++++++++++++++--- src/redismodule.h | 10 +++ tests/modules/Makefile | 3 +- tests/modules/scan.c | 62 ++++++++++++++++ tests/unit/moduleapi/scan.tcl | 18 +++++ 6 files changed, 215 insertions(+), 12 deletions(-) create mode 100644 tests/modules/scan.c create mode 100644 tests/unit/moduleapi/scan.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index e48535126..3eb6b21b2 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -22,4 +22,5 @@ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/hooks \ --single unit/moduleapi/misc \ --single unit/moduleapi/blockonkeys \ +--single unit/moduleapi/scan \ "${@}" diff --git a/src/module.c b/src/module.c index ad34e7b64..5758abbb6 100644 --- a/src/module.c +++ b/src/module.c @@ -1848,6 +1848,17 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } +static void initializeKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname, robj *value, int mode){ + kp->ctx = ctx; + kp->db = ctx->client->db; + kp->key = keyname; + incrRefCount(keyname); + kp->value = value; + kp->iter = NULL; + kp->mode = mode; + zsetKeyReset(kp); +} + /* Return an handle representing a Redis key, so that it is possible * to call other APIs with the key handle as argument to perform * operations on the key. @@ -1878,27 +1889,24 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { /* Setup the key handle. */ kp = zmalloc(sizeof(*kp)); - kp->ctx = ctx; - kp->db = ctx->client->db; - kp->key = keyname; - incrRefCount(keyname); - kp->value = value; - kp->iter = NULL; - kp->mode = mode; - zsetKeyReset(kp); + initializeKey(kp, ctx, keyname, value, mode); autoMemoryAdd(ctx,REDISMODULE_AM_KEY,kp); return (void*)kp; } -/* Close a key handle. */ -void RM_CloseKey(RedisModuleKey *key) { - if (key == NULL) return; +static void closeKeyInternal(RedisModuleKey *key) { int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); if ((key->mode & REDISMODULE_WRITE) && signal) signalModifiedKey(key->db,key->key); /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */ RM_ZsetRangeStop(key); decrRefCount(key->key); +} + +/* Close a key handle. */ +void RM_CloseKey(RedisModuleKey *key) { + if (key == NULL) return; + closeKeyInternal(key); autoMemoryFreed(key->ctx,REDISMODULE_AM_KEY,key); zfree(key); } @@ -5891,6 +5899,105 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } +/** + * Callback for scan implementation. + * + * The keyname is owned by the caller and need to be retained if used after this function. + * + * The kp is the data and provide using the best efforts approach, in some cases it might + * not be available (in such case it will be set to NULL) and it is the user responsibility + * to handle it. + * + * The kp (if given) is owned by the caller and will be free when the callback returns + * + */ +typedef void (*RedisModuleScanCB)(void *privdata, RedisModuleString* keyname, RedisModuleKey* key); + +typedef struct { + RedisModuleCtx *ctx; + void* user_data; + RedisModuleScanCB fn; +} ScanCBData; + +typedef struct RedisModuleCursor{ + int cursor; +}RedisModuleCursor; + +void ScanCallback(void *privdata, const dictEntry *de) { + ScanCBData *data = privdata; + sds key = dictGetKey(de); + robj* val = dictGetVal(de); + RedisModuleString *keyname = createObject(OBJ_STRING,sdsdup(key)); + + /* Setup the key handle. */ + RedisModuleKey kp = {0}; + initializeKey(&kp, data->ctx, keyname, val, REDISMODULE_READ); + + data->fn(data->user_data, keyname, &kp); + + closeKeyInternal(&kp); + decrRefCount(keyname); +} + +/** + * Create a new cursor to scan keys. + */ +RedisModuleCursor* RM_CursorCreate() { + RedisModuleCursor* cursor = zmalloc(sizeof(*cursor)); + cursor->cursor = 0; + return cursor; +} + +/** + * Restart an existing cursor. The keys will be rescanned. + */ +void RM_CursorRestart(RedisModuleCursor* cursor) { + cursor->cursor = 0; +} + +/** + * Destroy the cursor struct. + */ +void RM_CursorDestroy(RedisModuleCursor* cursor) { + zfree(cursor); +} + +/** + * Scan api that allows module writer to scan all the keys and value in redis. + * The way it should be used: + * Cursor* c = RedisModule_CursorCreate(); + * while(RedisModule_Scan(ctx, c, callback, privateData)); + * RedisModule_CursorDestroy(c); + * + * It is also possible to use this api from another thread such that the GIL only have to + * be acquired durring the actuall call to RM_Scan: + * Cursor* c = RedisModule_CursorCreate(); + * RedisModule_ThreadSafeCtxLock(ctx); + * while(RedisModule_Scan(ctx, c, callback, privateData)){ + * RedisModule_ThreadSafeCtxUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeCtxLock(ctx); + * } + * RedisModule_CursorDestroy(c); + * + * The function will return 1 if there is more elements to scan and 0 otherwise. + * It is also possible to restart and existing cursor using RM_CursorRestart + */ +int RM_Scan(RedisModuleCtx *ctx, RedisModuleCursor* cursor, RedisModuleScanCB fn, void* privdata) { + if(cursor->cursor == -1){ + return 0; + } + int ret = 1; + ScanCBData data = { ctx, privdata, fn }; + cursor->cursor = dictScan(ctx->client->db->dict, cursor->cursor, ScanCallback, NULL, &data); + if (cursor->cursor == 0){ + cursor->cursor = -1; + ret = 0; + } + return ret; +} + + /* -------------------------------------------------------------------------- * Module fork API * -------------------------------------------------------------------------- */ @@ -6969,6 +7076,10 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(SetLRUOrLFU); REGISTER_API(GetLRUOrLFU); REGISTER_API(BlockClientOnKeys); + REGISTER_API(Scan); + REGISTER_API(CursorCreate); + REGISTER_API(CursorDestroy); + REGISTER_API(CursorRestart); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); } diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..c74772d0f 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -392,6 +392,7 @@ typedef struct RedisModuleDictIter RedisModuleDictIter; typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx; typedef struct RedisModuleCommandFilter RedisModuleCommandFilter; typedef struct RedisModuleInfoCtx RedisModuleInfoCtx; +typedef struct RedisModuleCursor RedisModuleCursor; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -409,6 +410,7 @@ typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); +typedef void (*RedisModuleScanCB)(void *privdata, RedisModuleString* keyname, RedisModuleKey* key); #define REDISMODULE_TYPE_METHOD_VERSION 2 typedef struct RedisModuleTypeMethods { @@ -633,6 +635,10 @@ int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandF int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); +RedisModuleCursor* REDISMODULE_API_FUNC(RedisModule_CursorCreate)(); +void REDISMODULE_API_FUNC(RedisModule_CursorRestart)(RedisModuleCursor* cursor); +void REDISMODULE_API_FUNC(RedisModule_CursorDestroy)(RedisModuleCursor* cursor); +int REDISMODULE_API_FUNC(RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleCursor* cursor, RedisModuleScanCB fn, void* privdata); #endif #define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) @@ -842,6 +848,10 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(Fork); REDISMODULE_GET_API(ExitFromChild); REDISMODULE_GET_API(KillForkChild); + REDISMODULE_GET_API(Scan); + REDISMODULE_GET_API(CursorCreate); + REDISMODULE_GET_API(CursorRestart); + REDISMODULE_GET_API(CursorDestroy); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/tests/modules/Makefile b/tests/modules/Makefile index 9e27758a2..07c3cb829 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -19,7 +19,8 @@ TEST_MODULES = \ propagate.so \ misc.so \ hooks.so \ - blockonkeys.so + blockonkeys.so \ + scan.so .PHONY: all diff --git a/tests/modules/scan.c b/tests/modules/scan.c new file mode 100644 index 000000000..21071720a --- /dev/null +++ b/tests/modules/scan.c @@ -0,0 +1,62 @@ +#define REDISMODULE_EXPERIMENTAL_API +#include "redismodule.h" + +#include +#include +#include + +#define UNUSED(V) ((void) V) + +typedef struct scan_pd{ + size_t nkeys; + RedisModuleCtx *ctx; +} scan_pd; + +void scan_callback(void *privdata, RedisModuleString* keyname, RedisModuleKey* key){ + scan_pd* pd = privdata; + RedisModule_ReplyWithArray(pd->ctx, 2); + + RedisModule_ReplyWithString(pd->ctx, keyname); + if(key && RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_STRING){ + size_t len; + char * data = RedisModule_StringDMA(key, &len, REDISMODULE_READ); + RedisModule_ReplyWithStringBuffer(pd->ctx, data, len); + }else{ + RedisModule_ReplyWithNull(pd->ctx); + } + pd->nkeys++; +} + +int scan_keys_values(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + scan_pd pd = { + .nkeys = 0, + .ctx = ctx, + }; + + RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN); + + RedisModuleCursor* cursor = RedisModule_CursorCreate(); + while(RedisModule_Scan(ctx, cursor, scan_callback, &pd)); + RedisModule_CursorDestroy(cursor); + + RedisModule_ReplySetArrayLength(ctx, pd.nkeys); + return 0; +} + +int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + if (RedisModule_Init(ctx, "scan", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx, "scan.scankeysvalues", scan_keys_values, "", 0, 0, 0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + return REDISMODULE_OK; +} + + + + + diff --git a/tests/unit/moduleapi/scan.tcl b/tests/unit/moduleapi/scan.tcl new file mode 100644 index 000000000..5a77e8195 --- /dev/null +++ b/tests/unit/moduleapi/scan.tcl @@ -0,0 +1,18 @@ +set testmodule [file normalize tests/modules/scan.so] + +proc count_log_message {pattern} { + set result [exec grep -c $pattern < [srv 0 stdout]] +} + +start_server {tags {"modules"}} { + r module load $testmodule + + test {Module scan} { + # the module create a scan command which also return values + r set x 1 + r set y 2 + r set z 3 + lsort [r scan.scankeysvalues] + } {{x 1} {y 2} {z 3}} + +} \ No newline at end of file From 4c7b540ccd6adbda1ecd8f049a50bfac76ddde45 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 11 Nov 2019 13:30:37 +0200 Subject: [PATCH 269/320] Add RM_ScanKey to scan hash, set, zset, changes to RM_Scan API - Adding RM_ScanKey - Adding tests for RM_ScanKey - Refactoring RM_Scan API Changes in RM_Scan - cleanup in docs and coding convention - Moving out of experimantal Api - Adding ctx to scan callback - Dont use cursor of -1 as an indication of done (can be a valid cursor) - Set errno when returning 0 for various reasons - Rename Cursor to ScanCursor - Test filters key that are not strings, and opens a key if NULL --- src/module.c | 255 ++++++++++++++++++++++++++-------- src/redismodule.h | 23 +-- tests/modules/scan.c | 107 ++++++++++---- tests/unit/moduleapi/scan.tcl | 45 ++++-- 4 files changed, 323 insertions(+), 107 deletions(-) diff --git a/src/module.c b/src/module.c index 5758abbb6..b92a9e692 100644 --- a/src/module.c +++ b/src/module.c @@ -1848,7 +1848,8 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { return (retval == C_OK) ? REDISMODULE_OK : REDISMODULE_ERR; } -static void initializeKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname, robj *value, int mode){ +/* Initialize a RedisModuleKey struct */ +static void moduleInitKey(RedisModuleKey *kp, RedisModuleCtx *ctx, robj *keyname, robj *value, int mode){ kp->ctx = ctx; kp->db = ctx->client->db; kp->key = keyname; @@ -1889,12 +1890,13 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) { /* Setup the key handle. */ kp = zmalloc(sizeof(*kp)); - initializeKey(kp, ctx, keyname, value, mode); + moduleInitKey(kp, ctx, keyname, value, mode); autoMemoryAdd(ctx,REDISMODULE_AM_KEY,kp); return (void*)kp; } -static void closeKeyInternal(RedisModuleKey *key) { +/* Destroy a RedisModuleKey struct (freeing is the responsibility of the caller). */ +static void moduleCloseKey(RedisModuleKey *key) { int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx); if ((key->mode & REDISMODULE_WRITE) && signal) signalModifiedKey(key->db,key->key); @@ -1906,7 +1908,7 @@ static void closeKeyInternal(RedisModuleKey *key) { /* Close a key handle. */ void RM_CloseKey(RedisModuleKey *key) { if (key == NULL) return; - closeKeyInternal(key); + moduleCloseKey(key); autoMemoryFreed(key->ctx,REDISMODULE_AM_KEY,key); zfree(key); } @@ -5899,31 +5901,23 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } -/** - * Callback for scan implementation. - * - * The keyname is owned by the caller and need to be retained if used after this function. - * - * The kp is the data and provide using the best efforts approach, in some cases it might - * not be available (in such case it will be set to NULL) and it is the user responsibility - * to handle it. - * - * The kp (if given) is owned by the caller and will be free when the callback returns - * - */ -typedef void (*RedisModuleScanCB)(void *privdata, RedisModuleString* keyname, RedisModuleKey* key); +/* -------------------------------------------------------------------------- + * Scanning keyspace and hashes + * -------------------------------------------------------------------------- */ +typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); typedef struct { RedisModuleCtx *ctx; void* user_data; RedisModuleScanCB fn; } ScanCBData; -typedef struct RedisModuleCursor{ +typedef struct RedisModuleScanCursor{ int cursor; -}RedisModuleCursor; + int done; +}RedisModuleScanCursor; -void ScanCallback(void *privdata, const dictEntry *de) { +static void moduleScanCallback(void *privdata, const dictEntry *de) { ScanCBData *data = privdata; sds key = dictGetKey(de); robj* val = dictGetVal(de); @@ -5931,69 +5925,211 @@ void ScanCallback(void *privdata, const dictEntry *de) { /* Setup the key handle. */ RedisModuleKey kp = {0}; - initializeKey(&kp, data->ctx, keyname, val, REDISMODULE_READ); + moduleInitKey(&kp, data->ctx, keyname, val, REDISMODULE_READ); - data->fn(data->user_data, keyname, &kp); + data->fn(data->ctx, keyname, &kp, data->user_data); - closeKeyInternal(&kp); + moduleCloseKey(&kp); decrRefCount(keyname); } -/** - * Create a new cursor to scan keys. - */ -RedisModuleCursor* RM_CursorCreate() { - RedisModuleCursor* cursor = zmalloc(sizeof(*cursor)); +/* Create a new cursor to be used with RedisModule_Scan */ +RedisModuleScanCursor *RM_ScanCursorCreate() { + RedisModuleScanCursor* cursor = zmalloc(sizeof(*cursor)); cursor->cursor = 0; + cursor->done = 0; return cursor; } -/** - * Restart an existing cursor. The keys will be rescanned. - */ -void RM_CursorRestart(RedisModuleCursor* cursor) { +/* Restart an existing cursor. The keys will be rescanned. */ +void RM_ScanCursorRestart(RedisModuleScanCursor *cursor) { cursor->cursor = 0; + cursor->done = 0; } -/** - * Destroy the cursor struct. - */ -void RM_CursorDestroy(RedisModuleCursor* cursor) { +/* Destroy the cursor struct. */ +void RM_ScanCursorDestroy(RedisModuleScanCursor *cursor) { zfree(cursor); } -/** - * Scan api that allows module writer to scan all the keys and value in redis. +/* Scan api that allows a module to scan all the keys and value in the selected db. + * + * Callback for scan implementation. + * void scan_callback(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); + * - ctx - the redis module context provided to for the scan. + * - keyname - owned by the caller and need to be retained if used after this function. + * - key - holds info on the key and value, it is provided as best effort, in some cases it might + * be NULL, in which case the user should (can) use RedisModule_OpenKey (and CloseKey too). + * when it is provided, it is owned by the caller and will be free when the callback returns. + * - privdata - the user data provided to RedisModule_Scan. + * * The way it should be used: - * Cursor* c = RedisModule_CursorCreate(); + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); * while(RedisModule_Scan(ctx, c, callback, privateData)); - * RedisModule_CursorDestroy(c); + * RedisModule_ScanCursorDestroy(c); * - * It is also possible to use this api from another thread such that the GIL only have to - * be acquired durring the actuall call to RM_Scan: - * Cursor* c = RedisModule_CursorCreate(); - * RedisModule_ThreadSafeCtxLock(ctx); + * It is also possible to use this API from another thread while the lock is acquired durring + * the actuall call to RM_Scan: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); * while(RedisModule_Scan(ctx, c, callback, privateData)){ - * RedisModule_ThreadSafeCtxUnlock(ctx); + * RedisModule_ThreadSafeContextUnlock(ctx); * // do some background job - * RedisModule_ThreadSafeCtxLock(ctx); + * RedisModule_ThreadSafeContextLock(ctx); * } - * RedisModule_CursorDestroy(c); + * RedisModule_ScanCursorDestroy(c); * - * The function will return 1 if there is more elements to scan and 0 otherwise. - * It is also possible to restart and existing cursor using RM_CursorRestart - */ -int RM_Scan(RedisModuleCtx *ctx, RedisModuleCursor* cursor, RedisModuleScanCB fn, void* privdata) { - if(cursor->cursor == -1){ + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. */ +int RM_Scan(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata) { + if (cursor->done) { + errno = ENOENT; return 0; } int ret = 1; ScanCBData data = { ctx, privdata, fn }; - cursor->cursor = dictScan(ctx->client->db->dict, cursor->cursor, ScanCallback, NULL, &data); - if (cursor->cursor == 0){ - cursor->cursor = -1; + cursor->cursor = dictScan(ctx->client->db->dict, cursor->cursor, moduleScanCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; ret = 0; } + errno = 0; + return ret; +} + +typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata); +typedef struct { + RedisModuleKey *key; + void* user_data; + RedisModuleScanKeyCB fn; +} ScanKeyCBData; + +static void moduleScanKeyCallback(void *privdata, const dictEntry *de) { + ScanKeyCBData *data = privdata; + sds key = dictGetKey(de); + robj *o = data->key->value; + robj *field = createStringObject(key, sdslen(key)); + robj *value = NULL; + if (o->type == OBJ_SET) { + value = NULL; + } else if (o->type == OBJ_HASH) { + sds val = dictGetVal(de); + value = createStringObject(val, sdslen(val)); + } else if (o->type == OBJ_ZSET) { + double *val = (double*)dictGetVal(de); + value = createStringObjectFromLongDouble(*val, 0); + } + + data->fn(data->key, field, value, data->user_data); + decrRefCount(field); + if (value) decrRefCount(value); +} + +/* Scan api that allows a module to scan the elements in a hash, set or sorted set key + * + * Callback for scan implementation. + * void scan_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata); + * - key - the redis key context provided to for the scan. + * - field - field name, owned by the caller and need to be retained if used + * after this function. + * - value - value string or NULL for set type, owned by the caller and need to + * be retained if used after this function. + * - privdata - the user data provided to RedisModule_ScanKey. + * + * The way it should be used: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(key, c, callback, privateData)); + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * It is also possible to use this API from another thread while the lock is acquired durring + * the actuall call to RM_Scan, and re-opening the key each time: + * RedisModuleCursor *c = RedisModule_ScanCursorCreate(); + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * while(RedisModule_ScanKey(ctx, c, callback, privateData)){ + * RedisModule_CloseKey(key); + * RedisModule_ThreadSafeContextUnlock(ctx); + * // do some background job + * RedisModule_ThreadSafeContextLock(ctx); + * RedisModuleKey *key = RedisModule_OpenKey(...) + * } + * RedisModule_CloseKey(key); + * RedisModule_ScanCursorDestroy(c); + * + * The function will return 1 if there are more elements to scan and 0 otherwise, + * possibly setting errno if the call failed. + * It is also possible to restart and existing cursor using RM_CursorRestart. */ +int RM_ScanKey(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata) { + if (key == NULL || key->value == NULL) { + errno = EINVAL; + return 0; + } + dict *ht = NULL; + robj *o = key->value; + if (o->type == OBJ_SET) { + if (o->encoding == OBJ_ENCODING_HT) + ht = o->ptr; + } else if (o->type == OBJ_HASH) { + if (o->encoding == OBJ_ENCODING_HT) + ht = o->ptr; + } else if (o->type == OBJ_ZSET) { + if (o->encoding == OBJ_ENCODING_SKIPLIST) + ht = ((zset *)o->ptr)->dict; + } else { + errno = EINVAL; + return 0; + } + if (cursor->done) { + errno = ENOENT; + return 0; + } + int ret = 1; + if (ht) { + ScanKeyCBData data = { key, privdata, fn }; + cursor->cursor = dictScan(ht, cursor->cursor, moduleScanKeyCallback, NULL, &data); + if (cursor->cursor == 0) { + cursor->done = 1; + ret = 0; + } + } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_INTSET) { + int pos = 0; + int64_t ll; + while(intsetGet(o->ptr,pos++,&ll)) { + robj *field = createStringObjectFromLongLong(ll); + fn(key, field, NULL, privdata); + decrRefCount(field); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } else if (o->type == OBJ_HASH || o->type == OBJ_ZSET) { + unsigned char *p = ziplistIndex(o->ptr,0); + unsigned char *vstr; + unsigned int vlen; + long long vll; + while(p) { + ziplistGet(p,&vstr,&vlen,&vll); + robj *field = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + p = ziplistNext(o->ptr,p); + ziplistGet(p,&vstr,&vlen,&vll); + robj *value = (vstr != NULL) ? + createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll); + fn(key, field, value, privdata); + p = ziplistNext(o->ptr,p); + decrRefCount(field); + decrRefCount(value); + } + cursor->cursor = 1; + cursor->done = 1; + ret = 0; + } + errno = 0; return ret; } @@ -7076,10 +7212,11 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(SetLRUOrLFU); REGISTER_API(GetLRUOrLFU); REGISTER_API(BlockClientOnKeys); - REGISTER_API(Scan); - REGISTER_API(CursorCreate); - REGISTER_API(CursorDestroy); - REGISTER_API(CursorRestart); REGISTER_API(SignalKeyAsReady); REGISTER_API(GetBlockedClientReadyKey); + REGISTER_API(ScanCursorCreate); + REGISTER_API(ScanCursorDestroy); + REGISTER_API(ScanCursorRestart); + REGISTER_API(Scan); + REGISTER_API(ScanKey); } diff --git a/src/redismodule.h b/src/redismodule.h index c74772d0f..07e1452e1 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -392,7 +392,7 @@ typedef struct RedisModuleDictIter RedisModuleDictIter; typedef struct RedisModuleCommandFilterCtx RedisModuleCommandFilterCtx; typedef struct RedisModuleCommandFilter RedisModuleCommandFilter; typedef struct RedisModuleInfoCtx RedisModuleInfoCtx; -typedef struct RedisModuleCursor RedisModuleCursor; +typedef struct RedisModuleScanCursor RedisModuleScanCursor; typedef int (*RedisModuleCmdFunc)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc); typedef void (*RedisModuleDisconnectFunc)(RedisModuleCtx *ctx, RedisModuleBlockedClient *bc); @@ -410,7 +410,8 @@ typedef void (*RedisModuleTimerProc)(RedisModuleCtx *ctx, void *data); typedef void (*RedisModuleCommandFilterFunc) (RedisModuleCommandFilterCtx *filter); typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data); typedef void (*RedisModuleInfoFunc)(RedisModuleInfoCtx *ctx, int for_crash_report); -typedef void (*RedisModuleScanCB)(void *privdata, RedisModuleString* keyname, RedisModuleKey* key); +typedef void (*RedisModuleScanCB)(RedisModuleCtx *ctx, RedisModuleString *keyname, RedisModuleKey *key, void *privdata); +typedef void (*RedisModuleScanKeyCB)(RedisModuleKey *key, RedisModuleString *field, RedisModuleString *value, void *privdata); #define REDISMODULE_TYPE_METHOD_VERSION 2 typedef struct RedisModuleTypeMethods { @@ -590,6 +591,11 @@ int REDISMODULE_API_FUNC(RedisModule_GetLRUOrLFU)(RedisModuleKey *key, long long RedisModuleBlockedClient *REDISMODULE_API_FUNC(RedisModule_BlockClientOnKeys)(RedisModuleCtx *ctx, RedisModuleCmdFunc reply_callback, RedisModuleCmdFunc timeout_callback, void (*free_privdata)(RedisModuleCtx*,void*), long long timeout_ms, RedisModuleString **keys, int numkeys, void *privdata); void REDISMODULE_API_FUNC(RedisModule_SignalKeyAsReady)(RedisModuleCtx *ctx, RedisModuleString *key); RedisModuleString *REDISMODULE_API_FUNC(RedisModule_GetBlockedClientReadyKey)(RedisModuleCtx *ctx); +RedisModuleScanCursor *REDISMODULE_API_FUNC(RedisModule_ScanCursorCreate)(); +void REDISMODULE_API_FUNC(RedisModule_ScanCursorRestart)(RedisModuleScanCursor *cursor); +void REDISMODULE_API_FUNC(RedisModule_ScanCursorDestroy)(RedisModuleScanCursor *cursor); +int REDISMODULE_API_FUNC(RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleScanCursor *cursor, RedisModuleScanCB fn, void *privdata); +int REDISMODULE_API_FUNC(RedisModule_ScanKey)(RedisModuleKey *key, RedisModuleScanCursor *cursor, RedisModuleScanKeyCB fn, void *privdata); /* Experimental APIs */ #ifdef REDISMODULE_EXPERIMENTAL_API @@ -635,10 +641,6 @@ int REDISMODULE_API_FUNC(RedisModule_CommandFilterArgDelete)(RedisModuleCommandF int REDISMODULE_API_FUNC(RedisModule_Fork)(RedisModuleForkDoneHandler cb, void *user_data); int REDISMODULE_API_FUNC(RedisModule_ExitFromChild)(int retcode); int REDISMODULE_API_FUNC(RedisModule_KillForkChild)(int child_pid); -RedisModuleCursor* REDISMODULE_API_FUNC(RedisModule_CursorCreate)(); -void REDISMODULE_API_FUNC(RedisModule_CursorRestart)(RedisModuleCursor* cursor); -void REDISMODULE_API_FUNC(RedisModule_CursorDestroy)(RedisModuleCursor* cursor); -int REDISMODULE_API_FUNC(RedisModule_Scan)(RedisModuleCtx *ctx, RedisModuleCursor* cursor, RedisModuleScanCB fn, void* privdata); #endif #define RedisModule_IsAOFClient(id) ((id) == UINT64_MAX) @@ -805,6 +807,11 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(BlockClientOnKeys); REDISMODULE_GET_API(SignalKeyAsReady); REDISMODULE_GET_API(GetBlockedClientReadyKey); + REDISMODULE_GET_API(ScanCursorCreate); + REDISMODULE_GET_API(ScanCursorRestart); + REDISMODULE_GET_API(ScanCursorDestroy); + REDISMODULE_GET_API(Scan); + REDISMODULE_GET_API(ScanKey); #ifdef REDISMODULE_EXPERIMENTAL_API REDISMODULE_GET_API(GetThreadSafeContext); @@ -848,10 +855,6 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(Fork); REDISMODULE_GET_API(ExitFromChild); REDISMODULE_GET_API(KillForkChild); - REDISMODULE_GET_API(Scan); - REDISMODULE_GET_API(CursorCreate); - REDISMODULE_GET_API(CursorRestart); - REDISMODULE_GET_API(CursorDestroy); #endif if (RedisModule_IsModuleNameBusy && RedisModule_IsModuleNameBusy(name)) return REDISMODULE_ERR; diff --git a/tests/modules/scan.c b/tests/modules/scan.c index 21071720a..afede244b 100644 --- a/tests/modules/scan.c +++ b/tests/modules/scan.c @@ -1,62 +1,109 @@ -#define REDISMODULE_EXPERIMENTAL_API #include "redismodule.h" #include #include #include -#define UNUSED(V) ((void) V) +typedef struct { + size_t nkeys; +} scan_strings_pd; -typedef struct scan_pd{ - size_t nkeys; - RedisModuleCtx *ctx; -} scan_pd; +void scan_strings_callback(RedisModuleCtx *ctx, RedisModuleString* keyname, RedisModuleKey* key, void *privdata) { + scan_strings_pd* pd = privdata; + int was_opened = 0; + if (!key) { + key = RedisModule_OpenKey(ctx, keyname, REDISMODULE_READ); + was_opened = 1; + } -void scan_callback(void *privdata, RedisModuleString* keyname, RedisModuleKey* key){ - scan_pd* pd = privdata; - RedisModule_ReplyWithArray(pd->ctx, 2); - - RedisModule_ReplyWithString(pd->ctx, keyname); - if(key && RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_STRING){ + if (RedisModule_KeyType(key) == REDISMODULE_KEYTYPE_STRING) { size_t len; char * data = RedisModule_StringDMA(key, &len, REDISMODULE_READ); - RedisModule_ReplyWithStringBuffer(pd->ctx, data, len); - }else{ - RedisModule_ReplyWithNull(pd->ctx); + RedisModule_ReplyWithArray(ctx, 2); + RedisModule_ReplyWithString(ctx, keyname); + RedisModule_ReplyWithStringBuffer(ctx, data, len); + pd->nkeys++; } - pd->nkeys++; + if (was_opened) + RedisModule_CloseKey(key); } -int scan_keys_values(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +int scan_strings(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - scan_pd pd = { - .nkeys = 0, - .ctx = ctx, + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + scan_strings_pd pd = { + .nkeys = 0, }; RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN); - RedisModuleCursor* cursor = RedisModule_CursorCreate(); - while(RedisModule_Scan(ctx, cursor, scan_callback, &pd)); - RedisModule_CursorDestroy(cursor); + RedisModuleScanCursor* cursor = RedisModule_ScanCursorCreate(); + while(RedisModule_Scan(ctx, cursor, scan_strings_callback, &pd)); + RedisModule_ScanCursorDestroy(cursor); RedisModule_ReplySetArrayLength(ctx, pd.nkeys); - return 0; + return REDISMODULE_OK; +} + +typedef struct { + RedisModuleCtx *ctx; + size_t nreplies; +} scan_key_pd; + +void scan_key_callback(RedisModuleKey *key, RedisModuleString* field, RedisModuleString* value, void *privdata) { + REDISMODULE_NOT_USED(key); + scan_key_pd* pd = privdata; + RedisModule_ReplyWithArray(pd->ctx, 2); + RedisModule_ReplyWithString(pd->ctx, field); + if (value) + RedisModule_ReplyWithString(pd->ctx, value); + else + RedisModule_ReplyWithNull(pd->ctx); + pd->nreplies++; +} + +int scan_key(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) +{ + if (argc != 2) { + RedisModule_WrongArity(ctx); + return REDISMODULE_OK; + } + scan_key_pd pd = { + .ctx = ctx, + .nreplies = 0, + }; + + RedisModuleKey *key = RedisModule_OpenKey(ctx, argv[1], REDISMODULE_READ); + if (!key) { + RedisModule_ReplyWithError(ctx, "not found"); + return REDISMODULE_OK; + } + + RedisModule_ReplyWithArray(ctx, REDISMODULE_POSTPONED_ARRAY_LEN); + + RedisModuleScanCursor* cursor = RedisModule_ScanCursorCreate(); + while(RedisModule_ScanKey(key, cursor, scan_key_callback, &pd)); + RedisModule_ScanCursorDestroy(cursor); + + RedisModule_ReplySetArrayLength(ctx, pd.nreplies); + RedisModule_CloseKey(key); + return REDISMODULE_OK; } int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { - UNUSED(argv); - UNUSED(argc); + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); if (RedisModule_Init(ctx, "scan", 1, REDISMODULE_APIVER_1)== REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_CreateCommand(ctx, "scan.scankeysvalues", scan_keys_values, "", 0, 0, 0) == REDISMODULE_ERR) + if (RedisModule_CreateCommand(ctx, "scan.scan_strings", scan_strings, "", 0, 0, 0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx, "scan.scan_key", scan_key, "", 0, 0, 0) == REDISMODULE_ERR) return REDISMODULE_ERR; return REDISMODULE_OK; } - - - diff --git a/tests/unit/moduleapi/scan.tcl b/tests/unit/moduleapi/scan.tcl index 5a77e8195..de1672e0a 100644 --- a/tests/unit/moduleapi/scan.tcl +++ b/tests/unit/moduleapi/scan.tcl @@ -1,18 +1,47 @@ set testmodule [file normalize tests/modules/scan.so] -proc count_log_message {pattern} { - set result [exec grep -c $pattern < [srv 0 stdout]] -} - start_server {tags {"modules"}} { r module load $testmodule - test {Module scan} { - # the module create a scan command which also return values + test {Module scan keyspace} { + # the module create a scan command with filtering which also return values r set x 1 r set y 2 r set z 3 - lsort [r scan.scankeysvalues] + r hset h f v + lsort [r scan.scan_strings] } {{x 1} {y 2} {z 3}} -} \ No newline at end of file + test {Module scan hash ziplist} { + r hmset hh f1 v1 f2 v2 + lsort [r scan.scan_key hh] + } {{f1 v1} {f2 v2}} + + test {Module scan hash dict} { + r config set hash-max-ziplist-entries 2 + r hmset hh f3 v3 + lsort [r scan.scan_key hh] + } {{f1 v1} {f2 v2} {f3 v3}} + + test {Module scan zset ziplist} { + r zadd zz 1 f1 2 f2 + lsort [r scan.scan_key zz] + } {{f1 1} {f2 2}} + + test {Module scan zset dict} { + r config set zset-max-ziplist-entries 2 + r zadd zz 3 f3 + lsort [r scan.scan_key zz] + } {{f1 1} {f2 2} {f3 3}} + + test {Module scan set intset} { + r sadd ss 1 2 + lsort [r scan.scan_key ss] + } {{1 {}} {2 {}}} + + test {Module scan set dict} { + r config set set-max-intset-entries 2 + r sadd ss 3 + lsort [r scan.scan_key ss] + } {{1 {}} {2 {}} {3 {}}} +} From eac0e6094e68db542dce7db2037ddf5999b1b9ba Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 10 Nov 2019 09:38:50 +0200 Subject: [PATCH 270/320] Adjustments for active defrag defaults and tuning Reduce default minimum effort, so that when fragmentation is just detected, the impact on the latency will be minor. Reduce the default maximum effort, mainly to prevent a case were a sudden massive deletions, won't trigger an aggressive defrag that will cause latency. When activedefrag is disabled mid-run, reset the 'running' info field, and clear the scan cursor, so that when it'll be re-enabled, a new fresh scan will start. Clearing the 'running' variable is important since lowering the defragger tunables mid-scan won't help, the defragger only considers new threshold when a new scan starts, and during a scan it can only become more aggressive, (when more severe fragmentation is detected), it'll never go less aggressive. So by temporarily disabling activedefrag, one can lower th the tunables. Removing the experimantal warning. --- redis.conf | 16 +++++++--------- src/defrag.c | 44 ++++++++++++++++++++++++++++++-------------- src/server.c | 4 ++-- src/server.h | 4 ++-- 4 files changed, 41 insertions(+), 27 deletions(-) diff --git a/redis.conf b/redis.conf index 0ec3321a5..fb6f34b35 100644 --- a/redis.conf +++ b/redis.conf @@ -1606,10 +1606,6 @@ rdb-save-incremental-fsync yes ########################### ACTIVE DEFRAGMENTATION ####################### # -# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested -# even in production and manually tested by multiple engineers for some -# time. -# # What is active defragmentation? # ------------------------------- # @@ -1649,7 +1645,7 @@ rdb-save-incremental-fsync yes # a good idea to leave the defaults untouched. # Enabled active defragmentation -# activedefrag yes +# activedefrag no # Minimum amount of fragmentation waste to start active defrag # active-defrag-ignore-bytes 100mb @@ -1660,11 +1656,13 @@ rdb-save-incremental-fsync yes # Maximum percentage of fragmentation at which we use maximum effort # active-defrag-threshold-upper 100 -# Minimal effort for defrag in CPU percentage -# active-defrag-cycle-min 5 +# Minimal effort for defrag in CPU percentage, to be used when the lower +# threshold is reached +# active-defrag-cycle-min 1 -# Maximal effort for defrag in CPU percentage -# active-defrag-cycle-max 75 +# Maximal effort for defrag in CPU percentage, to be used when the upper +# threshold is reached +# active-defrag-cycle-max 25 # Maximum number of set/hash/zset/list fields that will be processed from # the main dictionary scan diff --git a/src/defrag.c b/src/defrag.c index e794c8e41..04e57955b 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -919,10 +919,12 @@ int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) { return 0; } +/* static variables serving defragLaterStep to continue scanning a key from were we stopped last time. */ +static sds defrag_later_current_key = NULL; +static unsigned long defrag_later_cursor = 0; + /* returns 0 if no more work needs to be been done, and 1 if time is up and more work is needed. */ int defragLaterStep(redisDb *db, long long endtime) { - static sds current_key = NULL; - static unsigned long cursor = 0; unsigned int iterations = 0; unsigned long long prev_defragged = server.stat_active_defrag_hits; unsigned long long prev_scanned = server.stat_active_defrag_scanned; @@ -930,16 +932,15 @@ int defragLaterStep(redisDb *db, long long endtime) { do { /* if we're not continuing a scan from the last call or loop, start a new one */ - if (!cursor) { + if (!defrag_later_cursor) { listNode *head = listFirst(db->defrag_later); /* Move on to next key */ - if (current_key) { - serverAssert(current_key == head->value); - sdsfree(head->value); + if (defrag_later_current_key) { + serverAssert(defrag_later_current_key == head->value); listDelNode(db->defrag_later, head); - cursor = 0; - current_key = NULL; + defrag_later_cursor = 0; + defrag_later_current_key = NULL; } /* stop if we reached the last one. */ @@ -948,21 +949,21 @@ int defragLaterStep(redisDb *db, long long endtime) { return 0; /* start a new key */ - current_key = head->value; - cursor = 0; + defrag_later_current_key = head->value; + defrag_later_cursor = 0; } /* each time we enter this function we need to fetch the key from the dict again (if it still exists) */ - dictEntry *de = dictFind(db->dict, current_key); + dictEntry *de = dictFind(db->dict, defrag_later_current_key); key_defragged = server.stat_active_defrag_hits; do { int quit = 0; - if (defragLaterItem(de, &cursor, endtime)) + if (defragLaterItem(de, &defrag_later_cursor, endtime)) quit = 1; /* time is up, we didn't finish all the work */ /* Don't start a new BIG key in this loop, this is because the * next key can be a list, and scanLaterList must be done in once cycle */ - if (!cursor) + if (!defrag_later_cursor) quit = 1; /* Once in 16 scan iterations, 512 pointer reallocations, or 64 fields @@ -982,7 +983,7 @@ int defragLaterStep(redisDb *db, long long endtime) { prev_defragged = server.stat_active_defrag_hits; prev_scanned = server.stat_active_defrag_scanned; } - } while(cursor); + } while(defrag_later_cursor); if(key_defragged != server.stat_active_defrag_hits) server.stat_active_defrag_key_hits++; else @@ -1039,6 +1040,21 @@ void activeDefragCycle(void) { mstime_t latency; int quit = 0; + if (!server.active_defrag_enabled) { + if (server.active_defrag_running) { + /* if active defrag was disabled mid-run, start from fresh next time. */ + server.active_defrag_running = 0; + if (db) + listEmpty(db->defrag_later); + defrag_later_current_key = NULL; + defrag_later_cursor = 0; + current_db = -1; + cursor = 0; + db = NULL; + } + return; + } + if (hasActiveChildProcess()) return; /* Defragging memory while there's a fork will just do damage. */ diff --git a/src/server.c b/src/server.c index 113d92cbb..f67fab632 100644 --- a/src/server.c +++ b/src/server.c @@ -1691,8 +1691,7 @@ void databasesCron(void) { } /* Defrag keys gradually. */ - if (server.active_defrag_enabled) - activeDefragCycle(); + activeDefragCycle(); /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad @@ -2854,6 +2853,7 @@ void initServer(void) { server.db[j].id = j; server.db[j].avg_ttl = 0; server.db[j].defrag_later = listCreate(); + listSetFreeMethod(server.db[j].defrag_later,(void (*)(void*))sdsfree); } evictionPoolAlloc(); /* Initialize the LRU keys pool. */ server.pubsub_channels = dictCreate(&keylistDictType,NULL); diff --git a/src/server.h b/src/server.h index a2c94a22a..cff45b5f5 100644 --- a/src/server.h +++ b/src/server.h @@ -174,8 +174,8 @@ typedef long long ustime_t; /* microsecond time type. */ #define CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER 10 /* don't defrag when fragmentation is below 10% */ #define CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER 100 /* maximum defrag force at 100% fragmentation */ #define CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES (100<<20) /* don't defrag if frag overhead is below 100mb */ -#define CONFIG_DEFAULT_DEFRAG_CYCLE_MIN 5 /* 5% CPU min (at lower threshold) */ -#define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 75 /* 75% CPU max (at upper threshold) */ +#define CONFIG_DEFAULT_DEFRAG_CYCLE_MIN 1 /* 1% CPU min (at lower threshold) */ +#define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 25 /* 25% CPU max (at upper threshold) */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ #define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ From 0b6feb152481760ae549a07a8a074dc5fc58b880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=96=9C=E6=AC=A2=E5=85=B0=E8=8A=B1=E5=B1=B1=E4=B8=98?= Date: Wed, 13 Nov 2019 10:14:45 +0800 Subject: [PATCH 271/320] Update adlist.h Update listGetFree keep format consistent --- src/adlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adlist.h b/src/adlist.h index c954fac87..28b9016ce 100644 --- a/src/adlist.h +++ b/src/adlist.h @@ -66,7 +66,7 @@ typedef struct list { #define listSetMatchMethod(l,m) ((l)->match = (m)) #define listGetDupMethod(l) ((l)->dup) -#define listGetFree(l) ((l)->free) +#define listGetFreeMethod(l) ((l)->free) #define listGetMatchMethod(l) ((l)->match) /* Prototypes */ From 181ae55753a670fba7d35202968ad22f62c097f5 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Wed, 13 Nov 2019 16:43:07 +0530 Subject: [PATCH 272/320] XADD with ID 0-0 stores an empty key Calling XADD with 0-0 or 0 would result in creating an empty key and storing it in the database. Even worse, because XADD will reply with error the action will not be replicated, creating a master-replica inconsistency --- src/t_stream.c | 8 ++++++++ tests/unit/type/stream.tcl | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 58b59f521..9bf87831f 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1220,6 +1220,14 @@ void xaddCommand(client *c) { return; } + /* Return ASAP if minimal ID (0-0) was given so we avoid possibly creating + * a new stream and have streamAppendItem fail, leaving an empty key in the + * database. */ + if (id_given && id.ms == 0 && id.seq == 0) { + addReplyError(c,"The ID specified in XADD must be greater than 0-0"); + return; + } + /* Lookup the stream at key. */ robj *o; stream *s; diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index aa9c5f3a9..a4431c654 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -123,6 +123,12 @@ start_server { assert {[r xlen mystream] == $j} } + test {XADD with ID 0-0} { + r DEL mystream + catch {r XADD mystream 0-0 k v} err + assert {[r EXISTS mystream] == 0} + } + test {XRANGE COUNT works as expected} { assert {[llength [r xrange mystream - + COUNT 10]] == 10} } From bf0931b57ca0b4419a7b2f8c8aeee4946042185c Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Fri, 23 Jun 2017 10:29:49 +0300 Subject: [PATCH 273/320] Add RM_ModuleTypeReplaceValue. This is a light-weight replace function, useful for use cases such as realloc()ing an existing value, etc. Using RM_ModuleTypeSetValue() in such cases is wasteful and complex as it attempts to delete the old value, call its destructor, etc. --- src/module.c | 29 ++++++++++++++++++++++++++++- src/redismodule.h | 2 ++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index ad34e7b64..0f22178c9 100644 --- a/src/module.c +++ b/src/module.c @@ -1950,7 +1950,7 @@ int RM_DeleteKey(RedisModuleKey *key) { return REDISMODULE_OK; } -/* If the key is open for writing, unlink it (that is delete it in a +/* If the key is open for writing, unlink it (that is delete it in a * non-blocking way, not reclaiming memory immediately) and setup the key to * accept new writes as an empty key (that will be created on demand). * On success REDISMODULE_OK is returned. If the key is not open for @@ -6768,6 +6768,32 @@ int RM_GetLRUOrLFU(RedisModuleKey *key, long long *lfu_freq, long long *lru_idle return REDISMODULE_OK; } +/* Replace the value assigned to a module type. + * + * The key must be open for writing, have an existing value, and have a moduleType + * that matches the one specified by the caller. + * + * Unlike RM_ModuleTypeSetValue() which will free the old value, this function + * simply swaps the old value with the new value. + * + * The function returns the old value, or NULL if any of the above conditions is + * not met. + */ +void *RM_ModuleTypeReplaceValue(RedisModuleKey *key, moduleType *mt, void *new_value) { + if (!(key->mode & REDISMODULE_WRITE) || key->iter) + return NULL; + if (!key->value || key->value->type != OBJ_MODULE) + return NULL; + + moduleValue *mv = key->value->ptr; + if (mv->type != mt) + return NULL; + + void *old_val = mv->value; + mv->value = new_value; + return old_val; +} + /* Register all the APIs we export. Keep this function at the end of the * file so that's easy to seek it to add new entries. */ void moduleRegisterCoreAPI(void) { @@ -6857,6 +6883,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(PoolAlloc); REGISTER_API(CreateDataType); REGISTER_API(ModuleTypeSetValue); + REGISTER_API(ModuleTypeReplaceValue); REGISTER_API(ModuleTypeGetType); REGISTER_API(ModuleTypeGetValue); REGISTER_API(IsIOError); diff --git a/src/redismodule.h b/src/redismodule.h index 728c7f584..8cf789fb0 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -517,6 +517,7 @@ int REDISMODULE_API_FUNC(RedisModule_GetContextFlags)(RedisModuleCtx *ctx); void *REDISMODULE_API_FUNC(RedisModule_PoolAlloc)(RedisModuleCtx *ctx, size_t bytes); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_CreateDataType)(RedisModuleCtx *ctx, const char *name, int encver, RedisModuleTypeMethods *typemethods); int REDISMODULE_API_FUNC(RedisModule_ModuleTypeSetValue)(RedisModuleKey *key, RedisModuleType *mt, void *value); +void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeReplaceValue)(RedisModuleKey *key, RedisModuleType *mt, void *new_value); RedisModuleType *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetType)(RedisModuleKey *key); void *REDISMODULE_API_FUNC(RedisModule_ModuleTypeGetValue)(RedisModuleKey *key); int REDISMODULE_API_FUNC(RedisModule_IsIOError)(RedisModuleIO *io); @@ -726,6 +727,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(PoolAlloc); REDISMODULE_GET_API(CreateDataType); REDISMODULE_GET_API(ModuleTypeSetValue); + REDISMODULE_GET_API(ModuleTypeReplaceValue); REDISMODULE_GET_API(ModuleTypeGetType); REDISMODULE_GET_API(ModuleTypeGetValue); REDISMODULE_GET_API(IsIOError); From c657b24c836370d58cebb4dce9e030d3a8ecd509 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 14 Nov 2019 09:09:10 +0200 Subject: [PATCH 274/320] module docs, missing LOADING flag --- src/module.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/module.c b/src/module.c index ad34e7b64..ceb18a557 100644 --- a/src/module.c +++ b/src/module.c @@ -1749,6 +1749,8 @@ int RM_GetSelectedDb(RedisModuleCtx *ctx) { * * REDISMODULE_CTX_FLAGS_OOM_WARNING: Less than 25% of memory remains before * reaching the maxmemory level. * + * * REDISMODULE_CTX_FLAGS_LOADING: Server is loading RDB/AOF + * * * REDISMODULE_CTX_FLAGS_REPLICA_IS_STALE: No active link with the master. * * * REDISMODULE_CTX_FLAGS_REPLICA_IS_CONNECTING: The replica is trying to From 10a834b48e8545f43a36f5932496aebc489674fd Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 14 Nov 2019 09:46:46 +0200 Subject: [PATCH 275/320] Slightly more efficient RM_ReplyWithEmptyString trimming talk about RESP protocol from API docs (should be independent to that anyway) --- src/module.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index ad34e7b64..6a6db3fe8 100644 --- a/src/module.c +++ b/src/module.c @@ -1389,7 +1389,7 @@ int RM_ReplyWithString(RedisModuleCtx *ctx, RedisModuleString *str) { int RM_ReplyWithEmptyString(RedisModuleCtx *ctx) { client *c = moduleGetReplyClient(ctx); if (c == NULL) return REDISMODULE_OK; - addReplyBulkCBuffer(c, "", 0); + addReply(c,shared.emptybulk); return REDISMODULE_OK; } @@ -1404,8 +1404,7 @@ int RM_ReplyWithVerbatimString(RedisModuleCtx *ctx, const char *buf, size_t len) return REDISMODULE_OK; } -/* Reply to the client with a NULL. In the RESP protocol a NULL is encoded - * as the string "$-1\r\n". +/* Reply to the client with a NULL. * * The function always returns REDISMODULE_OK. */ int RM_ReplyWithNull(RedisModuleCtx *ctx) { From df7f6bf4c299745d004016e3e815ab99956415c5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2019 12:48:54 +0100 Subject: [PATCH 276/320] Rax library updated. --- src/rax.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rax.c b/src/rax.c index be474b058..29b74ae90 100644 --- a/src/rax.c +++ b/src/rax.c @@ -1673,6 +1673,7 @@ int raxSeek(raxIterator *it, const char *op, unsigned char *ele, size_t len) { * node, but will be our match, representing the key "f". * * So in that case, we don't seek backward. */ + it->data = raxGetData(it->node); } else { if (gt && !raxIteratorNextStep(it,0)) return 0; if (lt && !raxIteratorPrevStep(it,0)) return 0; @@ -1791,7 +1792,7 @@ int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key if (eq && key_len == iter->key_len) return 1; else if (lt) return iter->key_len < key_len; else if (gt) return iter->key_len > key_len; - return 0; + else return 0; /* Avoid warning, just 'eq' is handled before. */ } else if (cmp > 0) { return gt ? 1 : 0; } else /* (cmp < 0) */ { From 688dbb4ae3b18264be307c2def05eb14c572c3b3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2019 18:27:37 +0100 Subject: [PATCH 277/320] Expire cycle: introduce the new state needed for the new algo. --- src/db.c | 2 ++ src/expire.c | 5 +++++ src/server.c | 2 ++ src/server.h | 5 ++--- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 8a7ea98ae..e47a4a681 100644 --- a/src/db.c +++ b/src/db.c @@ -1077,10 +1077,12 @@ int dbSwapDatabases(long id1, long id2) { db1->dict = db2->dict; db1->expires = db2->expires; db1->avg_ttl = db2->avg_ttl; + db1->expires_cursor = db2->expires_cursor; db2->dict = aux.dict; db2->expires = aux.expires; db2->avg_ttl = aux.avg_ttl; + db2->expires_cursor = aux.expires_cursor; /* Now we need to handle clients blocked on lists: as an effect * of swapping the two DBs, a client that was waiting for list diff --git a/src/expire.c b/src/expire.c index 598b27f96..3d0bae249 100644 --- a/src/expire.c +++ b/src/expire.c @@ -95,6 +95,10 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { * executed, where the time limit is a percentage of the REDIS_HZ period * as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. */ +#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ +#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds. */ +#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* Percentage of CPU to use. */ + void activeExpireCycle(int type) { /* This function has some global state in order to continue the work * incrementally across calls. */ @@ -231,6 +235,7 @@ void activeExpireCycle(int type) { } elapsed = ustime()-start; + server.stat_expire_cycle_time_used += elapsed; latencyAddSampleIfNeeded("expire-cycle",elapsed/1000); /* Update our estimate of keys existing but yet to be expired. diff --git a/src/server.c b/src/server.c index 113d92cbb..8c8b668de 100644 --- a/src/server.c +++ b/src/server.c @@ -2745,6 +2745,7 @@ void resetServerStats(void) { server.stat_expiredkeys = 0; server.stat_expired_stale_perc = 0; server.stat_expired_time_cap_reached_count = 0; + server.stat_expire_cycle_time_used = 0; server.stat_evictedkeys = 0; server.stat_keyspace_misses = 0; server.stat_keyspace_hits = 0; @@ -2848,6 +2849,7 @@ void initServer(void) { for (j = 0; j < server.dbnum; j++) { server.db[j].dict = dictCreate(&dbDictType,NULL); server.db[j].expires = dictCreate(&keyptrDictType,NULL); + server.db[j].expires_cursor = 0; server.db[j].blocking_keys = dictCreate(&keylistDictType,NULL); server.db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL); server.db[j].watched_keys = dictCreate(&keylistDictType,NULL); diff --git a/src/server.h b/src/server.h index a2c94a22a..2b7974594 100644 --- a/src/server.h +++ b/src/server.h @@ -180,9 +180,6 @@ typedef long long ustime_t; /* microsecond time type. */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ #define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ -#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ -#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ -#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* CPU max % for keys collection */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 @@ -721,6 +718,7 @@ typedef struct redisDb { dict *watched_keys; /* WATCHED keys for MULTI/EXEC CAS */ int id; /* Database ID */ long long avg_ttl; /* Average TTL, just for stats */ + unsigned long expires_cursor; /* Cursor of the active expire cycle. */ list *defrag_later; /* List of key names to attempt to defrag one by one, gradually. */ } redisDb; @@ -1167,6 +1165,7 @@ struct redisServer { long long stat_expiredkeys; /* Number of expired keys */ double stat_expired_stale_perc; /* Percentage of keys probably expired */ long long stat_expired_time_cap_reached_count; /* Early expire cylce stops.*/ + long long stat_expire_cycle_time_used; /* Cumulative microseconds used. */ long long stat_evictedkeys; /* Number of evicted keys (maxmemory) */ long long stat_keyspace_hits; /* Number of successful lookups of keys */ long long stat_keyspace_misses; /* Number of failed lookups of keys */ From 545109470ae3fb12b5eba4027474c97f71e13651 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 15 Nov 2019 10:38:55 +0100 Subject: [PATCH 278/320] Expire cycle: scan hash table buckets directly. --- src/expire.c | 93 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/src/expire.c b/src/expire.c index 3d0bae249..af9eaebf1 100644 --- a/src/expire.c +++ b/src/expire.c @@ -78,24 +78,40 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { * it will get more aggressive to avoid that too much memory is used by * keys that can be removed from the keyspace. * - * No more than CRON_DBS_PER_CALL databases are tested at every - * iteration. + * Every expire cycle tests multiple databases: the next call will start + * again from the next db, with the exception of exists for time limit: in that + * case we restart again from the last database we were processing. Anyway + * no more than CRON_DBS_PER_CALL databases are tested at every iteration. * - * This kind of call is used when Redis detects that timelimit_exit is - * true, so there is more work to do, and we do it more incrementally from - * the beforeSleep() function of the event loop. + * The function can perform more or less work, depending on the "type" + * argument. It can execute a "fast cycle" or a "slow cycle". The slow + * cycle is the main way we collect expired cycles: this happens with + * the "server.hz" frequency (usually 10 hertz). * - * Expire cycle type: + * However the slow cycle can exit for timeout, since it used too much time. + * For this reason the function is also invoked to perform a fast cycle + * at every event loop cycle, in the beforeSleep() function. The fast cycle + * will try to perform less work, but will do it much more often. + * + * The following are the details of the two expire cycles and their stop + * conditions: * * If type is ACTIVE_EXPIRE_CYCLE_FAST the function will try to run a * "fast" expire cycle that takes no longer than EXPIRE_FAST_CYCLE_DURATION * microseconds, and is not repeated again before the same amount of time. + * The cycle will also refuse to run at all if the latest slow cycle did not + * terminate because of a time limit condition. * * If type is ACTIVE_EXPIRE_CYCLE_SLOW, that normal expire cycle is * executed, where the time limit is a percentage of the REDIS_HZ period - * as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. */ + * as specified by the ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC define. In the + * fast cycle, the check of every database is interrupted once the number + * of already expired keys in the database is estimated to be lower than + * a given percentage, in order to avoid doing too much work to gain too + * little memory. + */ -#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ +#define ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP 20 /* HT buckets checked. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds. */ #define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* Percentage of CPU to use. */ @@ -152,7 +168,9 @@ void activeExpireCycle(int type) { long total_expired = 0; for (j = 0; j < dbs_per_call && timelimit_exit == 0; j++) { - int expired; + /* Expired and checked in a single loop. */ + unsigned long expired, sampled; + redisDb *db = server.db+(current_db % server.dbnum); /* Increment the DB now so we are sure if we run out of time @@ -176,8 +194,8 @@ void activeExpireCycle(int type) { slots = dictSlots(db->expires); now = mstime(); - /* When there are less than 1% filled slots getting random - * keys is expensive, so stop here waiting for better times... + /* When there are less than 1% filled slots, sampling the key + * space is expensive, so stop here waiting for better times... * The dictionary will be resized asap. */ if (num && slots > DICT_HT_INITIAL_SIZE && (num*100/slots < 1)) break; @@ -185,27 +203,47 @@ void activeExpireCycle(int type) { /* The main collection cycle. Sample random keys among keys * with an expire set, checking for expired ones. */ expired = 0; + sampled = 0; ttl_sum = 0; ttl_samples = 0; - if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) - num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP; + if (num > ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP) + num = ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP; - while (num--) { - dictEntry *de; - long long ttl; + /* Here we access the low level representation of the hash table + * for speed concerns: this makes this code coupled with dict.c, + * but it hardly changed in ten years. */ + while (sampled < num) { + for (int table = 0; table < 2; table++) { + if (table == 1 && !dictIsRehashing(db->expires)) break; - if ((de = dictGetRandomKey(db->expires)) == NULL) break; - ttl = dictGetSignedIntegerVal(de)-now; - if (activeExpireCycleTryExpire(db,de,now)) expired++; - if (ttl > 0) { - /* We want the average TTL of keys yet not expired. */ - ttl_sum += ttl; - ttl_samples++; + unsigned long idx = db->expires_cursor; + idx &= db->expires->ht[table].sizemask; + dictEntry *de = db->expires->ht[table].table[idx]; + long long ttl; + + /* Scan the current bucket of the current table. */ + while(de) { + /* Get the next entry now since this entry may get + * deleted. */ + dictEntry *e = de; + de = de->next; + + ttl = dictGetSignedIntegerVal(e)-now; + if (activeExpireCycleTryExpire(db,e,now)) expired++; + if (ttl > 0) { + /* We want the average TTL of keys yet + * not expired. */ + ttl_sum += ttl; + ttl_samples++; + } + sampled++; + } } - total_sampled++; + db->expires_cursor++; } total_expired += expired; + total_sampled += sampled; /* Update the average TTL stats for this database. */ if (ttl_samples) { @@ -229,9 +267,10 @@ void activeExpireCycle(int type) { break; } } - /* We don't repeat the cycle if there are less than 25% of keys - * found expired in the current DB. */ - } while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4); + /* We don't repeat the cycle for the current database if there are + * less than 25% of keys found expired in the current DB. */ + // printf("[%d] Expired %d, sampled %d\n", type, (int) expired, (int) sampled); + } while (expired > sampled/4); } elapsed = ustime()-start; From eecf9c87c0a113c3a74ad74ecf2ccb6cd480b873 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 15 Nov 2019 11:08:03 +0100 Subject: [PATCH 279/320] Expire cycle: tollerate less stale keys, expire cycle CPU in INFO. --- src/expire.c | 31 +++++++++++++++++++------------ src/server.c | 2 ++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/expire.c b/src/expire.c index af9eaebf1..9cb4c4a0a 100644 --- a/src/expire.c +++ b/src/expire.c @@ -111,9 +111,11 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { * little memory. */ -#define ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP 20 /* HT buckets checked. */ +#define ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP 20 /* Keys for each DB loop. */ #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds. */ -#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* Percentage of CPU to use. */ +#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* Max % of CPU to use. */ +#define ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE 10 /* % of stale keys after which + we do extra efforts. */ void activeExpireCycle(int type) { /* This function has some global state in order to continue the work @@ -133,10 +135,15 @@ void activeExpireCycle(int type) { if (type == ACTIVE_EXPIRE_CYCLE_FAST) { /* Don't start a fast cycle if the previous cycle did not exit - * for time limit. Also don't repeat a fast cycle for the same period + * for time limit, unless the percentage of estimated stale keys is + * too high. Also never repeat a fast cycle for the same period * as the fast cycle total duration itself. */ - if (!timelimit_exit) return; - if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) return; + if (!timelimit_exit && server.stat_expired_stale_perc < + ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE) return; + + if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) + return; + last_fast_cycle = start; } @@ -150,8 +157,8 @@ void activeExpireCycle(int type) { if (dbs_per_call > server.dbnum || timelimit_exit) dbs_per_call = server.dbnum; - /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time - * per iteration. Since this function gets called with a frequency of + /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU + * time per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100; @@ -207,8 +214,8 @@ void activeExpireCycle(int type) { ttl_sum = 0; ttl_samples = 0; - if (num > ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP) - num = ACTIVE_EXPIRE_CYCLE_BUCKETS_PER_LOOP; + if (num > ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP) + num = ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP; /* Here we access the low level representation of the hash table * for speed concerns: this makes this code coupled with dict.c, @@ -268,9 +275,9 @@ void activeExpireCycle(int type) { } } /* We don't repeat the cycle for the current database if there are - * less than 25% of keys found expired in the current DB. */ - // printf("[%d] Expired %d, sampled %d\n", type, (int) expired, (int) sampled); - } while (expired > sampled/4); + * an acceptable amount of stale keys (logically expired but yet + * not reclained). */ + } while ((expired*100/sampled) > ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE); } elapsed = ustime()-start; diff --git a/src/server.c b/src/server.c index 8c8b668de..f834119ee 100644 --- a/src/server.c +++ b/src/server.c @@ -4270,6 +4270,7 @@ sds genRedisInfoString(char *section) { "expired_keys:%lld\r\n" "expired_stale_perc:%.2f\r\n" "expired_time_cap_reached_count:%lld\r\n" + "expire_cycle_cpu_milliseconds:%lld\r\n" "evicted_keys:%lld\r\n" "keyspace_hits:%lld\r\n" "keyspace_misses:%lld\r\n" @@ -4297,6 +4298,7 @@ sds genRedisInfoString(char *section) { server.stat_expiredkeys, server.stat_expired_stale_perc*100, server.stat_expired_time_cap_reached_count, + server.stat_expire_cycle_time_used/1000, server.stat_evictedkeys, server.stat_keyspace_hits, server.stat_keyspace_misses, From d4f31e0d5f108c9f4526839c24486c2164f6fd6c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2019 11:30:05 +0100 Subject: [PATCH 280/320] Expire cycle: introduce configurable effort. --- src/expire.c | 37 ++++++++++++++++++++++++++++--------- src/server.c | 1 + src/server.h | 2 ++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/expire.c b/src/expire.c index 9cb4c4a0a..dd8a42726 100644 --- a/src/expire.c +++ b/src/expire.c @@ -109,6 +109,9 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { * of already expired keys in the database is estimated to be lower than * a given percentage, in order to avoid doing too much work to gain too * little memory. + * + * The configured expire "effort" will modify the baseline parameters in + * order to do more work in both the fast and slow expire cycles. */ #define ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP 20 /* Keys for each DB loop. */ @@ -118,6 +121,21 @@ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { we do extra efforts. */ void activeExpireCycle(int type) { + /* Adjust the running parameters according to the configured expire + * effort. The default effort is 1, and the maximum configurable effort + * is 10. */ + unsigned long + effort = server.active_expire_effort, + config_keys_per_loop = ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP + + ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP/4*effort, + config_cycle_fast_duration = ACTIVE_EXPIRE_CYCLE_FAST_DURATION * + ACTIVE_EXPIRE_CYCLE_FAST_DURATION/4*effort, + config_cycle_slow_time_perc = ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC + + 2*effort, + config_cycle_acceptable_stale = ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE- + effort; + if (config_cycle_acceptable_stale < 1) config_cycle_acceptable_stale = 1; + /* This function has some global state in order to continue the work * incrementally across calls. */ static unsigned int current_db = 0; /* Last DB tested. */ @@ -138,10 +156,11 @@ void activeExpireCycle(int type) { * for time limit, unless the percentage of estimated stale keys is * too high. Also never repeat a fast cycle for the same period * as the fast cycle total duration itself. */ - if (!timelimit_exit && server.stat_expired_stale_perc < - ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE) return; + if (!timelimit_exit && + server.stat_expired_stale_perc < config_cycle_acceptable_stale) + return; - if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) + if (start < last_fast_cycle + (long long)config_cycle_fast_duration*2) return; last_fast_cycle = start; @@ -157,16 +176,16 @@ void activeExpireCycle(int type) { if (dbs_per_call > server.dbnum || timelimit_exit) dbs_per_call = server.dbnum; - /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU + /* We can use at max 'config_cycle_slow_time_perc' percentage of CPU * time per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ - timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100; + timelimit = config_cycle_slow_time_perc*1000000/server.hz/100; timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; if (type == ACTIVE_EXPIRE_CYCLE_FAST) - timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */ + timelimit = config_cycle_fast_duration; /* in microseconds. */ /* Accumulate some global stats as we expire keys, to have some idea * about the number of keys that are already logically expired, but still @@ -214,8 +233,8 @@ void activeExpireCycle(int type) { ttl_sum = 0; ttl_samples = 0; - if (num > ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP) - num = ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP; + if (num > config_keys_per_loop) + num = config_keys_per_loop; /* Here we access the low level representation of the hash table * for speed concerns: this makes this code coupled with dict.c, @@ -277,7 +296,7 @@ void activeExpireCycle(int type) { /* We don't repeat the cycle for the current database if there are * an acceptable amount of stale keys (logically expired but yet * not reclained). */ - } while ((expired*100/sampled) > ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE); + } while ((expired*100/sampled) > config_cycle_acceptable_stale); } elapsed = ustime()-start; diff --git a/src/server.c b/src/server.c index f834119ee..c811b869d 100644 --- a/src/server.c +++ b/src/server.c @@ -2294,6 +2294,7 @@ void initServerConfig(void) { server.maxidletime = CONFIG_DEFAULT_CLIENT_TIMEOUT; server.tcpkeepalive = CONFIG_DEFAULT_TCP_KEEPALIVE; server.active_expire_enabled = 1; + server.active_expire_effort = CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT; server.jemalloc_bg_thread = 1; server.active_defrag_enabled = CONFIG_DEFAULT_ACTIVE_DEFRAG; server.active_defrag_ignore_bytes = CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES; diff --git a/src/server.h b/src/server.h index 2b7974594..51dce955d 100644 --- a/src/server.h +++ b/src/server.h @@ -179,6 +179,7 @@ typedef long long ustime_t; /* microsecond time type. */ #define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ #define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ #define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ +#define CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT 1 /* From 1 to 10. */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 @@ -1204,6 +1205,7 @@ struct redisServer { int maxidletime; /* Client timeout in seconds */ int tcpkeepalive; /* Set SO_KEEPALIVE if non-zero. */ int active_expire_enabled; /* Can be disabled for testing purposes. */ + int active_expire_effort; /* From 1 (default) to 10, active effort. */ int active_defrag_enabled; int jemalloc_bg_thread; /* Enable jemalloc background thread */ size_t active_defrag_ignore_bytes; /* minimum amount of fragmentation waste to start active defrag */ From c7d07138c7eac143be3dbb12adab74c09b9e29c8 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2019 11:33:40 +0100 Subject: [PATCH 281/320] Expire cycle: make expire effort configurable. --- src/config.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 505dabc9c..de8e8d8cf 100644 --- a/src/config.c +++ b/src/config.c @@ -256,7 +256,7 @@ void loadServerConfigFromString(char *config) { for (configYesNo *config = configs_yesno; config->name != NULL; config++) { if ((!strcasecmp(argv[0],config->name) || (config->alias && !strcasecmp(argv[0],config->alias))) && - (argc == 2)) + (argc == 2)) { if ((*(config->config) = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -580,6 +580,14 @@ void loadServerConfigFromString(char *config) { err = "active-defrag-max-scan-fields must be positive"; goto loaderr; } + } else if (!strcasecmp(argv[0],"active-expire-effort") && argc == 2) { + server.active_expire_effort = atoi(argv[1]); + if (server.active_expire_effort < 1 || + server.active_expire_effort > 10) + { + err = "active-expire-effort must be between 1 and 10"; + goto loaderr; + } } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { server.hash_max_ziplist_entries = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { @@ -1165,6 +1173,8 @@ void configSetCommand(client *c) { "active-defrag-cycle-max",server.active_defrag_cycle_max,1,99) { } config_set_numerical_field( "active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,1,LONG_MAX) { + } config_set_numerical_field( + "active-expire-effort",server.active_expire_effort,1,10) { } config_set_numerical_field( "auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,INT_MAX){ } config_set_numerical_field( @@ -1478,6 +1488,7 @@ void configGetCommand(client *c) { config_get_numerical_field("active-defrag-cycle-min",server.active_defrag_cycle_min); config_get_numerical_field("active-defrag-cycle-max",server.active_defrag_cycle_max); config_get_numerical_field("active-defrag-max-scan-fields",server.active_defrag_max_scan_fields); + config_get_numerical_field("active-expire-effort",server.active_expire_effort); config_get_numerical_field("auto-aof-rewrite-percentage", server.aof_rewrite_perc); config_get_numerical_field("auto-aof-rewrite-min-size", @@ -2327,6 +2338,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"active-defrag-cycle-min",server.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN); rewriteConfigNumericalOption(state,"active-defrag-cycle-max",server.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX); rewriteConfigNumericalOption(state,"active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS); + rewriteConfigNumericalOption(state,"active-expire-effort",server.active_expire_effort,CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT); rewriteConfigYesNoOption(state,"appendonly",server.aof_enabled,0); rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC); From 833b43ceff2e098c0971b57667b9718abc8d23ae Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2019 11:43:42 +0100 Subject: [PATCH 282/320] Expire cycle: fix parameters computation. --- src/expire.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/expire.c b/src/expire.c index dd8a42726..ea7e2b456 100644 --- a/src/expire.c +++ b/src/expire.c @@ -125,16 +125,15 @@ void activeExpireCycle(int type) { * effort. The default effort is 1, and the maximum configurable effort * is 10. */ unsigned long - effort = server.active_expire_effort, + effort = server.active_expire_effort-1, /* Rescale from 0 to 9. */ config_keys_per_loop = ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP + ACTIVE_EXPIRE_CYCLE_KEYS_PER_LOOP/4*effort, - config_cycle_fast_duration = ACTIVE_EXPIRE_CYCLE_FAST_DURATION * + config_cycle_fast_duration = ACTIVE_EXPIRE_CYCLE_FAST_DURATION + ACTIVE_EXPIRE_CYCLE_FAST_DURATION/4*effort, config_cycle_slow_time_perc = ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC + 2*effort, config_cycle_acceptable_stale = ACTIVE_EXPIRE_CYCLE_ACCEPTABLE_STALE- effort; - if (config_cycle_acceptable_stale < 1) config_cycle_acceptable_stale = 1; /* This function has some global state in order to continue the work * incrementally across calls. */ From 2772453296258178a1206cabd3538053d6f75648 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2019 17:47:19 +0100 Subject: [PATCH 283/320] Expire cycle: set a buckets limit as well. --- src/expire.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/expire.c b/src/expire.c index ea7e2b456..b4ab9ab18 100644 --- a/src/expire.c +++ b/src/expire.c @@ -237,8 +237,18 @@ void activeExpireCycle(int type) { /* Here we access the low level representation of the hash table * for speed concerns: this makes this code coupled with dict.c, - * but it hardly changed in ten years. */ - while (sampled < num) { + * but it hardly changed in ten years. + * + * Note that certain places of the hash table may be empty, + * so we want also a stop condition about the number of + * buckets that we scanned. However scanning for free buckets + * is very fast: we are in the cache line scanning a sequential + * array of NULL pointers, so we can scan a lot more buckets + * than keys in the same time. */ + long max_buckets = num*20; + long checked_buckets = 0; + + while (sampled < num && checked_buckets < max_buckets) { for (int table = 0; table < 2; table++) { if (table == 1 && !dictIsRehashing(db->expires)) break; @@ -248,6 +258,7 @@ void activeExpireCycle(int type) { long long ttl; /* Scan the current bucket of the current table. */ + checked_buckets++; while(de) { /* Get the next entry now since this entry may get * deleted. */ From 25415849e95f2148c4f4bfe6d01ada1a64cd162b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2019 18:11:38 +0100 Subject: [PATCH 284/320] Expire cycle: document expire effort in redis.conf. --- redis.conf | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/redis.conf b/redis.conf index 0ec3321a5..39e21b5e7 100644 --- a/redis.conf +++ b/redis.conf @@ -813,11 +813,11 @@ replica-priority 100 # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory # is reached. You can select among five behaviors: # -# volatile-lru -> Evict using approximated LRU among the keys with an expire set. +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. # allkeys-lru -> Evict any key using approximated LRU. -# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. # allkeys-lfu -> Evict any key using approximated LFU. -# volatile-random -> Remove a random key among the ones with an expire set. +# volatile-random -> Remove a random key having an expire set. # allkeys-random -> Remove a random key, any key. # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) # noeviction -> Don't evict anything, just return an error on write operations. @@ -872,6 +872,23 @@ replica-priority 100 # # replica-ignore-maxmemory yes +# Redis reclaims expired keys in two ways: upon access when those keys are +# found to be expired, and also in background, in what is called the +# "active expire key". The key space is slowly and interactively scanned +# looking for expired keys to reclaim, so that it is possible to free memory +# of keys that are expired and will never be accessed again in a short time. +# +# The default effort of the expire cycle will try to avoid having more than +# ten percent of expired keys still in memory, and will try to avoid consuming +# more than 25% of total memory and to add latency to the system. However +# it is possible to increase the expire "effort" that is normally set to +# "1", to a greater value, up to the value "10". At its maximum value the +# system will use more CPU, longer cycles (and technically may introduce +# more latency), and will tollerate less already expired keys still present +# in the system. It's a tradeoff betweeen memory, CPU and latecy. +# +# active-expire-effort 1 + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking From d4aa06002c2057d7abc506b8696256d4f6b02a1e Mon Sep 17 00:00:00 2001 From: Daniel Dai <764122422@qq.com> Date: Tue, 19 Nov 2019 00:02:45 -0500 Subject: [PATCH 285/320] fix move command --- src/db.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/db.c b/src/db.c index e47a4a681..0f8eaefa3 100644 --- a/src/db.c +++ b/src/db.c @@ -1034,6 +1034,13 @@ void moveCommand(client *c) { /* OK! key moved, free the entry in the source DB */ dbDelete(src,c->argv[1]); + signalModifiedKey(src,c->argv[1]); + signalModifiedKey(dst,c->argv[1]); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_from",c->argv[1],src->id); + notifyKeyspaceEvent(NOTIFY_GENERIC, + "move_to",c->argv[1],dst->id); + server.dirty++; addReply(c,shared.cone); } From 270a013db5857b848868c4d9a0aa1872978ae051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=96=9C=E6=AC=A2=E5=85=B0=E8=8A=B1=E5=B1=B1=E4=B8=98?= Date: Tue, 19 Nov 2019 17:23:47 +0800 Subject: [PATCH 286/320] Update mkreleasehdr.sh fix date +%s errata --- src/mkreleasehdr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index e6d558b17..236c26c2b 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -3,7 +3,7 @@ GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` if [ -n "$SOURCE_DATE_EPOCH" ]; then - BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u %s) + BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u +%s) fi test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ From 6b6eb37a8dff35d08b6f9ae437ed9120caed40de Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2019 11:05:55 +0100 Subject: [PATCH 287/320] Remove additional space from comment. --- src/t_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 9bf87831f..0b07d7110 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1220,7 +1220,7 @@ void xaddCommand(client *c) { return; } - /* Return ASAP if minimal ID (0-0) was given so we avoid possibly creating + /* Return ASAP if minimal ID (0-0) was given so we avoid possibly creating * a new stream and have streamAppendItem fail, leaving an empty key in the * database. */ if (id_given && id.ms == 0 && id.seq == 0) { From 3d4a44bd0b21d1ad9d0d8252a625a9a9f1d20664 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Tue, 19 Nov 2019 12:10:48 +0200 Subject: [PATCH 288/320] Improve RM_Call() errno classification. RM_Call() will now use EBADF and ENONET in addition to EINVAL in order to provide more information about errors (i.e. when return value is NULL). --- src/module.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index ad34e7b64..8796a7ab1 100644 --- a/src/module.c +++ b/src/module.c @@ -3110,7 +3110,9 @@ fmterr: * On success a RedisModuleCallReply object is returned, otherwise * NULL is returned and errno is set to the following values: * - * EINVAL: command non existing, wrong arity, wrong format specifier. + * EBADF: wrong format specifier. + * EINVAL: wrong command arity. + * ENOENT: command does not exist. * EPERM: operation in Cluster instance with key in non local slot. * * This API is documented here: https://redis.io/topics/modules-intro @@ -3142,7 +3144,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch /* We handle the above format error only when the client is setup so that * we can free it normally. */ if (argv == NULL) { - errno = EINVAL; + errno = EBADF; goto cleanup; } @@ -3154,7 +3156,7 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch */ cmd = lookupCommand(c->argv[0]->ptr); if (!cmd) { - errno = EINVAL; + errno = ENOENT; goto cleanup; } c->cmd = c->lastcmd = cmd; From 63c25b90f4115e1ad1283c7213a5c0cbd7e6b346 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2019 11:23:43 +0100 Subject: [PATCH 289/320] Fix patch provided in #6554. --- src/blocked.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index dea4cc57a..47bb290a4 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -514,6 +514,13 @@ void handleClientsBlockedOnKeys(void) { * we can safely call signalKeyAsReady() against this key. */ dictDelete(rl->db->ready_keys,rl->key); + /* Even if we are not inside call(), increment the call depth + * in order to make sure that keys are expired against a fixed + * reference time, and not against the wallclock time. This + * way we can lookup an object multiple times (BRPOPLPUSH does + * that) without the risk of it being freed in the second + * lookup, invalidating the first one. + * See https://github.com/antirez/redis/pull/6554. */ server.call_depth++; updateCachedTime(0); @@ -533,7 +540,7 @@ void handleClientsBlockedOnKeys(void) { serveClientsBlockedOnKeyByModule(rl); } - server.call_depth++; + server.call_depth--; /* Free this item. */ decrRefCount(rl->key); From 0f0e8b10c6c5ed63dcd2b01c617159dcd7481b09 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2019 11:28:04 +0100 Subject: [PATCH 290/320] Rename var to fixed_time_expire now that is more general. --- src/blocked.c | 5 ++--- src/db.c | 2 +- src/server.c | 6 +++--- src/server.h | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 47bb290a4..20c0e760a 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -521,7 +521,7 @@ void handleClientsBlockedOnKeys(void) { * that) without the risk of it being freed in the second * lookup, invalidating the first one. * See https://github.com/antirez/redis/pull/6554. */ - server.call_depth++; + server.fixed_time_expire++; updateCachedTime(0); /* Serve clients blocked on list key. */ @@ -539,8 +539,7 @@ void handleClientsBlockedOnKeys(void) { * module is trying to accomplish right now. */ serveClientsBlockedOnKeyByModule(rl); } - - server.call_depth--; + server.fixed_time_expire--; /* Free this item. */ decrRefCount(rl->key); diff --git a/src/db.c b/src/db.c index e47a4a681..aac049eb7 100644 --- a/src/db.c +++ b/src/db.c @@ -1220,7 +1220,7 @@ int keyIsExpired(redisDb *db, robj *key) { * may re-open the same key multiple times, can invalidate an already * open object in a next call, if the next call will see the key expired, * while the first did not. */ - else if (server.call_depth > 0) { + else if (server.fixed_time_expire > 0) { now = server.mstime; } /* For the other cases, we want to use the most fresh time we have. */ diff --git a/src/server.c b/src/server.c index c811b869d..2e9a329dd 100644 --- a/src/server.c +++ b/src/server.c @@ -2788,7 +2788,7 @@ void initServer(void) { server.hz = server.config_hz; server.pid = getpid(); server.current_client = NULL; - server.call_depth = 0; + server.fixed_time_expire = 0; server.clients = listCreate(); server.clients_index = raxNew(); server.clients_to_close = listCreate(); @@ -3262,7 +3262,7 @@ void call(client *c, int flags) { int client_old_flags = c->flags; struct redisCommand *real_cmd = c->cmd; - server.call_depth++; + server.fixed_time_expire++; /* Sent the command to clients in MONITOR mode, only if the commands are * not generated from reading an AOF. */ @@ -3389,7 +3389,7 @@ void call(client *c, int flags) { trackingRememberKeys(caller); } - server.call_depth--; + server.fixed_time_expire--; server.stat_numcommands++; } diff --git a/src/server.h b/src/server.h index 51dce955d..5da7b4960 100644 --- a/src/server.h +++ b/src/server.h @@ -1134,7 +1134,7 @@ struct redisServer { list *clients_pending_read; /* Client has pending read socket buffers. */ list *slaves, *monitors; /* List of slaves and MONITORs */ client *current_client; /* Current client executing the command. */ - long call_depth; /* call() re-entering count. */ + long fixed_time_expire; /* If > 0, expire keys against server.mstime. */ rax *clients_index; /* Active clients dictionary by client ID. */ int clients_paused; /* True if clients are currently paused */ mstime_t clients_pause_end_time; /* Time when we undo clients_paused */ From fa60228c08e7e6d98b2cd858cec4580008114f22 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2019 11:49:05 +0100 Subject: [PATCH 291/320] Fix stream test after addition of 0-0 ID test. --- tests/unit/type/stream.tcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index a4431c654..656bac5de 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -124,9 +124,9 @@ start_server { } test {XADD with ID 0-0} { - r DEL mystream - catch {r XADD mystream 0-0 k v} err - assert {[r EXISTS mystream] == 0} + r DEL otherstream + catch {r XADD otherstream 0-0 k v} err + assert {[r EXISTS otherstream] == 0} } test {XRANGE COUNT works as expected} { From b5e76fc58e76a96ec3b63d7c3ff22e864ffadc19 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2019 11:56:02 +0100 Subject: [PATCH 292/320] Simplify PR #6551 implementation. --- src/sentinel.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index d5f22b97f..42c4d2467 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -465,12 +465,6 @@ struct redisCommand sentinelcmds[] = { {"hello",helloCommand,-2,"no-script fast",0,NULL,0,0,0,0,0} }; -/* List of client types that are killed when an instance becomes a slave */ -const char* killedClientTypes[] = { - "normal", - "pubsub" -}; - /* This function overwrites a few normal Redis config default with Sentinel * specific defaults. */ void initSentinelConfig(void) { @@ -3955,7 +3949,6 @@ char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t epoch) { int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { char portstr[32]; int retval; - unsigned int curType; ll2string(portstr,sizeof(portstr),port); @@ -4000,11 +3993,11 @@ int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { * an issue because CLIENT is variadic command, so Redis will not * recognized as a syntax error, and the transaction will not fail (but * only the unsupported command will fail). */ - for (curType = 0; curType < sizeof(killedClientTypes)/sizeof(killedClientTypes[0]); ++curType) { + for (int type = 0; type < 2; type++) { retval = redisAsyncCommand(ri->link->cc, sentinelDiscardReplyCallback, ri, "%s KILL TYPE %s", sentinelInstanceMapCommand(ri,"CLIENT"), - killedClientTypes[curType]); + type == 0 ? "normal" : "pubsub"); if (retval == C_ERR) return retval; ri->link->pending_commands++; } From 58b256bd46dbbe95d9776bcf85d7a04a008a98b8 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 19 Nov 2019 14:53:39 +0200 Subject: [PATCH 293/320] try to fix an unstable test (module hook for loading progress) there were two lssues, one is taht BGREWRITEAOF failed since the initial one was still in progress the solution for this one is to enable appendonly from the server startup so there's no initial aofrw. the other problem was 0 loading progress events, theory is that on some platforms a sleep of 1 will cause a much greater delay due to the context switch, but on other platform it doesn't. in theory a sleep of 100 micro for 1k keys whould take 100ms, and with hz of 500 we should be gettering 50 events (one every 2ms). in practise it doesn't work like that, so trying to find a sleep that would be long enough but still not cause the test to take too long. --- tests/unit/moduleapi/hooks.tcl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/moduleapi/hooks.tcl b/tests/unit/moduleapi/hooks.tcl index cbca9c3eb..bf2b9010b 100644 --- a/tests/unit/moduleapi/hooks.tcl +++ b/tests/unit/moduleapi/hooks.tcl @@ -1,9 +1,7 @@ set testmodule [file normalize tests/modules/hooks.so] tags "modules" { - start_server {} { - r module load $testmodule - r config set appendonly yes + start_server [list overrides [list loadmodule "$testmodule" appendonly yes]] { test {Test clients connection / disconnection hooks} { for {set j 0} {$j < 2} {incr j} { @@ -44,15 +42,19 @@ tags "modules" { r set "bar$j" x } # set some configs that will cause many loading progress events during aof loading - r config set key-load-delay 1 + r config set key-load-delay 500 r config set dynamic-hz no r config set hz 500 r DEBUG LOADAOF assert_equal [r hooks.event_last loading-aof-start] 0 assert_equal [r hooks.event_last loading-end] 0 assert {[r hooks.event_count loading-rdb-start] == 0} - assert {[r hooks.event_count loading-progress-rdb] >= 2} ;# comes from the preamble section - assert {[r hooks.event_count loading-progress-aof] >= 2} + assert_lessthan 2 [r hooks.event_count loading-progress-rdb] ;# comes from the preamble section + assert_lessthan 2 [r hooks.event_count loading-progress-aof] + if {$::verbose} { + puts "rdb progress events [r hooks.event_count loading-progress-rdb]" + puts "aof progress events [r hooks.event_count loading-progress-aof]" + } } # undo configs before next test r config set dynamic-hz yes From c7adfe68f9b307ca501ec15ba38259f55709e0ba Mon Sep 17 00:00:00 2001 From: Daniel Dai <764122422@qq.com> Date: Tue, 19 Nov 2019 20:14:59 -0500 Subject: [PATCH 294/320] fix typo --- tests/unit/scripting.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index b3e1c48e6..2543a0377 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -536,7 +536,7 @@ foreach cmdrepl {0 1} { start_server {tags {"scripting repl"}} { start_server {} { if {$cmdrepl == 1} { - set rt "(commmands replication)" + set rt "(commands replication)" } else { set rt "(scripts replication)" r debug lua-always-replicate-commands 1 From faeed15627e65094b9878276dd544013f67e8a15 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 19 Jul 2019 10:15:35 -0700 Subject: [PATCH 295/320] Refactored renaming types in config --- src/config.c | 1182 ++++++++++++++++++++++---------------------------- 1 file changed, 514 insertions(+), 668 deletions(-) diff --git a/src/config.c b/src/config.c index de8e8d8cf..f6440272e 100644 --- a/src/config.c +++ b/src/config.c @@ -107,46 +107,70 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { /* Configuration values that require no special handling to set, get, load or * rewrite. */ -typedef struct configYesNo { +typedef struct boolConfigData { + int *config; /* The pointer to the server config this value is stored in */ + const int default_value; /* The default value of the config on rewrite */ +} boolConfigData; + +typedef struct stringConfigData { + char **config; /* The pointer to the server config this value is stored in */ + const char *default_value; /* The default value of the config on rewrite */ +} stringConfigData; + +typedef struct enumConfigData { + int *config; /* The pointer to the server config this value is stored in */ + configEnum *enum_value; /* The underlying enum type this data represents */ + const int default_value; /* The default value of the config on rewrite */ +} enumConfigData; + +typedef enum numericType { + NUMERIC_TYPE_INT, + NUMERIC_TYPE_LONG_LONG, + NUMERIC_TYPE_UNSIGNED_LONG, + NUMERIC_TYPE_SIZE_T +} numericType; + +typedef struct numericConfigData { + union { + int *i; + long long *ll; + unsigned long *ul; + size_t *st; + } config; /* The pointer to the numeric config this value is stored in */ + int is_memory; /* Indicates if this value can be loaded as a memory value */ + numericType numeric_type; /* An enum indicating the type of this value */ + long long lower_bound; /* The lower bound of this numeric value */ + long long upper_bound; /* The upper bound of this numeric value */ + const long long default_value; /* The default value of the config on rewrite */ +} numericConfigData; + +typedef union typeData { + boolConfigData yesno; + stringConfigData string; + enumConfigData enumd; + numericConfigData numeric; +} typeData; + +typedef struct typeInterface { + /* Called on server start, should return 1 on success, 0 on error and should set err */ + int (*load)(typeData data, sds *argc, int argv, char **err); + /* Called on CONFIG SET, returns 1 on success, 0 on error */ + int (*set)(typeData data, sds value); + /* Called on CONFIG GET, required to add output to the client */ + void (*get)(client *c, typeData data); + /* Called on CONFIG REWRITE, required to rewrite the config state */ + void (*rewrite)(typeData data, const char *name, struct rewriteConfigState *state); +} typeInterface; + +typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ - int *config; /* The pointer to the server config this value is stored in */ const int modifiable; /* Can this value be updated by CONFIG SET? */ - const int default_value; /* The default value of the config on rewrite */ -} configYesNo; + typeInterface interface; /* The function points that define the type interface */ + typeData data; /* The type specific data exposed used by the interface */ +} standardConfig; -configYesNo configs_yesno[] = { - /* Non-Modifiable */ - {"rdbchecksum",NULL,&server.rdb_checksum,0,CONFIG_DEFAULT_RDB_CHECKSUM}, - {"daemonize",NULL,&server.daemonize,0,0}, - {"io-threads-do-reads",NULL,&server.io_threads_do_reads, 0, CONFIG_DEFAULT_IO_THREADS_DO_READS}, - {"always-show-logo",NULL,&server.always_show_logo,0,CONFIG_DEFAULT_ALWAYS_SHOW_LOGO}, - /* Modifiable */ - {"protected-mode",NULL,&server.protected_mode,1,CONFIG_DEFAULT_PROTECTED_MODE}, - {"rdbcompression",NULL,&server.rdb_compression,1,CONFIG_DEFAULT_RDB_COMPRESSION}, - {"activerehashing",NULL,&server.activerehashing,1,CONFIG_DEFAULT_ACTIVE_REHASHING}, - {"stop-writes-on-bgsave-error",NULL,&server.stop_writes_on_bgsave_err,1,CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR}, - {"dynamic-hz",NULL,&server.dynamic_hz,1,CONFIG_DEFAULT_DYNAMIC_HZ}, - {"lazyfree-lazy-eviction",NULL,&server.lazyfree_lazy_eviction,1,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION}, - {"lazyfree-lazy-expire",NULL,&server.lazyfree_lazy_expire,1,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE}, - {"lazyfree-lazy-server-del",NULL,&server.lazyfree_lazy_server_del,1,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL}, - {"repl-disable-tcp-nodelay",NULL,&server.repl_disable_tcp_nodelay,1,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY}, - {"repl-diskless-sync",NULL,&server.repl_diskless_sync,1,CONFIG_DEFAULT_REPL_DISKLESS_SYNC}, - {"gopher-enabled",NULL,&server.gopher_enabled,1,CONFIG_DEFAULT_GOPHER_ENABLED}, - {"aof-rewrite-incremental-fsync",NULL,&server.aof_rewrite_incremental_fsync,1,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC}, - {"no-appendfsync-on-rewrite",NULL,&server.aof_no_fsync_on_rewrite,1,CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE}, - {"cluster-require-full-coverage",NULL,&server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE}, - {"rdb-save-incremental-fsync",NULL,&server.rdb_save_incremental_fsync,1,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC}, - {"aof-load-truncated",NULL,&server.aof_load_truncated,1,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED}, - {"aof-use-rdb-preamble",NULL,&server.aof_use_rdb_preamble,1,CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE}, - {"cluster-replica-no-failover","cluster-slave-no-failover",&server.cluster_slave_no_failover,1,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER}, - {"replica-lazy-flush","slave-lazy-flush",&server.repl_slave_lazy_flush,1,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH}, - {"replica-serve-stale-data","slave-serve-stale-data",&server.repl_serve_stale_data,1,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA}, - {"replica-read-only","slave-read-only",&server.repl_slave_ro,1,CONFIG_DEFAULT_SLAVE_READ_ONLY}, - {"replica-ignore-maxmemory","slave-ignore-maxmemory",&server.repl_slave_ignore_maxmemory,1,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY}, - {"jemalloc-bg-thread",NULL,&server.jemalloc_bg_thread,1,1}, - {NULL, NULL, 0, 0} -}; +standardConfig configs[]; /*----------------------------------------------------------------------------- * Enum access functions @@ -220,7 +244,7 @@ void queueLoadModule(sds path, sds *argv, int argc) { } void loadServerConfigFromString(char *config) { - const char *err = NULL; + char *err = NULL; int linenum = 0, totlines, i; int slaveof_linenum = 0; sds *lines; @@ -253,14 +277,14 @@ void loadServerConfigFromString(char *config) { /* Iterate the configs that are standard */ int match = 0; - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { + for (standardConfig *config = configs; config->name != NULL; config++) { if ((!strcasecmp(argv[0],config->name) || - (config->alias && !strcasecmp(argv[0],config->alias))) && - (argc == 2)) + (config->alias && !strcasecmp(argv[0],config->alias)))) { - if ((*(config->config) = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; + if (!config->interface.load(config->data, argv, argc, &err)) { + goto loaderr; } + match = 1; break; } @@ -272,27 +296,7 @@ void loadServerConfigFromString(char *config) { } /* Execute config directives */ - if (!strcasecmp(argv[0],"timeout") && argc == 2) { - server.maxidletime = atoi(argv[1]); - if (server.maxidletime < 0) { - err = "Invalid timeout value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { - server.tcpkeepalive = atoi(argv[1]); - if (server.tcpkeepalive < 0) { - err = "Invalid tcp-keepalive value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"port") && argc == 2) { - server.port = atoi(argv[1]); - if (server.port < 0 || server.port > 65535) { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tcp-backlog") && argc == 2) { - server.tcp_backlog = atoi(argv[1]); - if (server.tcp_backlog < 0) { - err = "Invalid backlog value"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + if (!strcasecmp(argv[0],"bind") && argc >= 2) { int j, addresses = argc-1; if (addresses > CONFIG_BINDADDR_MAX) { @@ -301,8 +305,6 @@ void loadServerConfigFromString(char *config) { for (j = 0; j < addresses; j++) server.bindaddr[j] = zstrdup(argv[j+1]); server.bindaddr_count = addresses; - } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { - server.unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { errno = 0; server.unixsocketperm = (mode_t)strtol(argv[1], NULL, 8); @@ -326,13 +328,6 @@ void loadServerConfigFromString(char *config) { argv[1], strerror(errno)); exit(1); } - } else if (!strcasecmp(argv[0],"loglevel") && argc == 2) { - server.verbosity = configEnumGetValue(loglevel_enum,argv[1]); - if (server.verbosity == INT_MIN) { - err = "Invalid log level. " - "Must be one of debug, verbose, notice, warning"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; @@ -349,9 +344,6 @@ void loadServerConfigFromString(char *config) { } fclose(logfp); } - } else if (!strcasecmp(argv[0],"aclfile") && argc == 2) { - zfree(server.acl_filename); - server.acl_filename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) { if ((server.syslog_enabled = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -359,23 +351,6 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"syslog-ident") && argc == 2) { if (server.syslog_ident) zfree(server.syslog_ident); server.syslog_ident = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - server.syslog_facility = - configEnumGetValue(syslog_facility_enum,argv[1]); - if (server.syslog_facility == INT_MIN) { - err = "Invalid log facility. Must be one of USER or between LOCAL0-LOCAL7"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"databases") && argc == 2) { - server.dbnum = atoi(argv[1]); - if (server.dbnum < 1) { - err = "Invalid number of databases"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"io-threads") && argc == 2) { - server.io_threads_num = atoi(argv[1]); - if (server.io_threads_num < 1 || server.io_threads_num > 512) { - err = "Invalid number of I/O threads"; goto loaderr; - } } else if (!strcasecmp(argv[0],"include") && argc == 2) { loadServerConfig(argv[1],NULL); } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { @@ -385,68 +360,14 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { server.maxmemory = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { - server.maxmemory_policy = - configEnumGetValue(maxmemory_policy_enum,argv[1]); - if (server.maxmemory_policy == INT_MIN) { - err = "Invalid maxmemory policy"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory-samples") && argc == 2) { - server.maxmemory_samples = atoi(argv[1]); - if (server.maxmemory_samples <= 0) { - err = "maxmemory-samples must be 1 or greater"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"proto-max-bulk-len")) && argc == 2) { - server.proto_max_bulk_len = memtoll(argv[1],NULL); - } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { - server.client_max_querybuf_len = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"lfu-log-factor") && argc == 2) { - server.lfu_log_factor = atoi(argv[1]); - if (server.lfu_log_factor < 0) { - err = "lfu-log-factor must be 0 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lfu-decay-time") && argc == 2) { - server.lfu_decay_time = atoi(argv[1]); - if (server.lfu_decay_time < 0) { - err = "lfu-decay-time must be 0 or greater"; - goto loaderr; - } + } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { + server.client_max_querybuf_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; server.masterhost = sdsnew(argv[1]); server.masterport = atoi(argv[2]); server.repl_state = REPL_STATE_CONNECT; - } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || - !strcasecmp(argv[0],"repl-ping-replica-period")) && - argc == 2) - { - server.repl_ping_slave_period = atoi(argv[1]); - if (server.repl_ping_slave_period <= 0) { - err = "repl-ping-replica-period must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { - server.repl_timeout = atoi(argv[1]); - if (server.repl_timeout <= 0) { - err = "repl-timeout must be 1 or greater"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-load") && argc==2) { - server.repl_diskless_load = configEnumGetValue(repl_diskless_load_enum,argv[1]); - if (server.repl_diskless_load == INT_MIN) { - err = "argument must be 'disabled', 'on-empty-db', 'swapdb' or 'flushdb'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"repl-diskless-sync-delay") && argc==2) { - server.repl_diskless_sync_delay = atoi(argv[1]); - if (server.repl_diskless_sync_delay < 0) { - err = "repl-diskless-sync-delay can't be negative"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { long long size = memtoll(argv[1],NULL); if (size <= 0) { @@ -460,12 +381,6 @@ void loadServerConfigFromString(char *config) { err = "repl-backlog-ttl can't be negative "; goto loaderr; } - } else if (!strcasecmp(argv[0],"masteruser") && argc == 2) { - zfree(server.masteruser); - server.masteruser = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { - zfree(server.masterauth); - server.masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; } else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) { if ((server.active_defrag_enabled = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -491,36 +406,6 @@ void loadServerConfigFromString(char *config) { } zfree(server.aof_filename); server.aof_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"appendfsync") && argc == 2) { - server.aof_fsync = configEnumGetValue(aof_fsync_enum,argv[1]); - if (server.aof_fsync == INT_MIN) { - err = "argument must be 'no', 'always' or 'everysec'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-percentage") && - argc == 2) - { - server.aof_rewrite_perc = atoi(argv[1]); - if (server.aof_rewrite_perc < 0) { - err = "Invalid negative percentage for AOF auto rewrite"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && - argc == 2) - { - server.aof_rewrite_min_size = memtoll(argv[1],NULL); - } else if (!strcasecmp(argv[0],"rdb-key-save-delay") && argc==2) { - server.rdb_key_save_delay = atoi(argv[1]); - if (server.rdb_key_save_delay < 0) { - err = "rdb-key-save-delay can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"key-load-delay") && argc==2) { - server.key_load_delay = atoi(argv[1]); - if (server.key_load_delay < 0) { - err = "key-load-delay can't be negative"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; @@ -532,9 +417,6 @@ void loadServerConfigFromString(char *config) { sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { - zfree(server.pidfile); - server.pidfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { if (!pathIsBaseName(argv[1])) { err = "dbfilename can't be a path, just a filename"; @@ -542,76 +424,10 @@ void loadServerConfigFromString(char *config) { } zfree(server.rdb_filename); server.rdb_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"active-defrag-threshold-lower") && argc == 2) { - server.active_defrag_threshold_lower = atoi(argv[1]); - if (server.active_defrag_threshold_lower < 0 || - server.active_defrag_threshold_lower > 1000) { - err = "active-defrag-threshold-lower must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-threshold-upper") && argc == 2) { - server.active_defrag_threshold_upper = atoi(argv[1]); - if (server.active_defrag_threshold_upper < 0 || - server.active_defrag_threshold_upper > 1000) { - err = "active-defrag-threshold-upper must be between 0 and 1000"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-ignore-bytes") && argc == 2) { - server.active_defrag_ignore_bytes = memtoll(argv[1], NULL); - if (server.active_defrag_ignore_bytes <= 0) { - err = "active-defrag-ignore-bytes must above 0"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-min") && argc == 2) { - server.active_defrag_cycle_min = atoi(argv[1]); - if (server.active_defrag_cycle_min < 1 || server.active_defrag_cycle_min > 99) { - err = "active-defrag-cycle-min must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-cycle-max") && argc == 2) { - server.active_defrag_cycle_max = atoi(argv[1]); - if (server.active_defrag_cycle_max < 1 || server.active_defrag_cycle_max > 99) { - err = "active-defrag-cycle-max must be between 1 and 99"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-defrag-max-scan-fields") && argc == 2) { - server.active_defrag_max_scan_fields = strtoll(argv[1],NULL,10); - if (server.active_defrag_max_scan_fields < 1) { - err = "active-defrag-max-scan-fields must be positive"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"active-expire-effort") && argc == 2) { - server.active_expire_effort = atoi(argv[1]); - if (server.active_expire_effort < 1 || - server.active_expire_effort > 10) - { - err = "active-expire-effort must be between 1 and 10"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { - server.hash_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hash-max-ziplist-value") && argc == 2) { - server.hash_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-bytes") && argc == 2) { - server.stream_node_max_bytes = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { - server.stream_node_max_entries = atoi(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { /* DEAD OPTION */ - } else if (!strcasecmp(argv[0],"list-max-ziplist-size") && argc == 2) { - server.list_max_ziplist_size = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"list-compress-depth") && argc == 2) { - server.list_compress_depth = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"set-max-intset-entries") && argc == 2) { - server.set_max_intset_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-entries") && argc == 2) { - server.zset_max_ziplist_entries = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"zset-max-ziplist-value") && argc == 2) { - server.zset_max_ziplist_value = memtoll(argv[1], NULL); - } else if (!strcasecmp(argv[0],"hll-sparse-max-bytes") && argc == 2) { - server.hll_sparse_max_bytes = memtoll(argv[1], NULL); } else if (!strcasecmp(argv[0],"rename-command") && argc == 3) { struct redisCommand *cmd = lookupCommand(argv[1]); int retval; @@ -643,80 +459,6 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(server.cluster_configfile); server.cluster_configfile = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-ip") && argc == 2) { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"cluster-announce-port") && argc == 2) { - server.cluster_announce_port = atoi(argv[1]); - if (server.cluster_announce_port < 0 || - server.cluster_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-announce-bus-port") && - argc == 2) - { - server.cluster_announce_bus_port = atoi(argv[1]); - if (server.cluster_announce_bus_port < 0 || - server.cluster_announce_bus_port > 65535) - { - err = "Invalid port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-node-timeout") && argc == 2) { - server.cluster_node_timeout = strtoll(argv[1],NULL,10); - if (server.cluster_node_timeout <= 0) { - err = "cluster node timeout must be 1 or greater"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-migration-barrier") - && argc == 2) - { - server.cluster_migration_barrier = atoi(argv[1]); - if (server.cluster_migration_barrier < 0) { - err = "cluster migration barrier must zero or positive"; - goto loaderr; - } - } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || - !strcasecmp(argv[0],"cluster-replica-validity-factor")) - && argc == 2) - { - server.cluster_slave_validity_factor = atoi(argv[1]); - if (server.cluster_slave_validity_factor < 0) { - err = "cluster replica validity factor must be zero or positive"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { - server.lua_time_limit = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { - server.lua_always_replicate_commands = yesnotoi(argv[1]); - if (server.lua_always_replicate_commands == -1) { - err = "argument must be 'yes' or 'no'"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && - argc == 2) - { - server.slowlog_log_slower_than = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"latency-monitor-threshold") && - argc == 2) - { - server.latency_monitor_threshold = strtoll(argv[1],NULL,10); - if (server.latency_monitor_threshold < 0) { - err = "The latency threshold can't be negative"; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"slowlog-max-len") && argc == 2) { - server.slowlog_max_len = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"tracking-table-max-fill") && - argc == 2) - { - server.tracking_table_max_fill = strtoll(argv[1],NULL,10); - if (server.tracking_table_max_fill > 100 || - server.tracking_table_max_fill < 0) - { - err = "The tracking table fill percentage must be an " - "integer between 0 and 100"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { @@ -739,24 +481,6 @@ void loadServerConfigFromString(char *config) { server.client_obuf_limits[class].hard_limit_bytes = hard; server.client_obuf_limits[class].soft_limit_bytes = soft; server.client_obuf_limits[class].soft_limit_seconds = soft_seconds; - } else if ((!strcasecmp(argv[0],"slave-priority") || - !strcasecmp(argv[0],"replica-priority")) && argc == 2) - { - server.slave_priority = atoi(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-ip") || - !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = zstrdup(argv[1]); - } else if ((!strcasecmp(argv[0],"slave-announce-port") || - !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) - { - server.slave_announce_port = atoi(argv[1]); - if (server.slave_announce_port < 0 || - server.slave_announce_port > 65535) - { - err = "Invalid port"; goto loaderr; - } } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) { @@ -779,15 +503,6 @@ void loadServerConfigFromString(char *config) { goto loaderr; } server.notify_keyspace_events = flags; - } else if (!strcasecmp(argv[0],"supervised") && argc == 2) { - server.supervised_mode = - configEnumGetValue(supervised_mode_enum,argv[1]); - - if (server.supervised_mode == INT_MIN) { - err = "Invalid option for 'supervised'. " - "Allowed values: 'upstart', 'systemd', 'auto', or 'no'"; - goto loaderr; - } } else if (!strcasecmp(argv[0],"user") && argc >= 2) { int argc_err; if (ACLAppendUserForLoading(argv,argc,&argc_err) == C_ERR) { @@ -935,17 +650,11 @@ void loadServerConfig(char *filename, char *options) { if (err || ll < 0) goto badfmt; \ _var = ll; -#define config_set_enum_field(_name,_var,_enumvar) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name)) { \ - int enumval = configEnumGetValue(_enumvar,o->ptr); \ - if (enumval == INT_MIN) goto badfmt; \ - _var = enumval; - #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { -#define config_set_special_field_with_alias(_name1,_name2) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name1) || \ +#define config_set_special_field_with_alias(_name,_name2) \ + } else if (!strcasecmp(c->argv[2]->ptr,_name) || \ !strcasecmp(c->argv[2]->ptr,_name2)) { #define config_set_else } else @@ -959,13 +668,13 @@ void configSetCommand(client *c) { o = c->argv[3]; /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { + for (standardConfig *config = configs; config->name != NULL; config++) { if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { - int yn = yesnotoi(o->ptr); - if (yn == -1) goto badfmt; - *(config->config) = yn; + if (!config->interface.set(config->data,o->ptr)) { + goto badfmt; + } addReply(c,shared.ok); return; } @@ -989,15 +698,6 @@ void configSetCommand(client *c) { sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } config_set_special_field("masteruser") { - zfree(server.masteruser); - server.masteruser = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("masterauth") { - zfree(server.masterauth); - server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } config_set_special_field("cluster-announce-ip") { - zfree(server.cluster_announce_ip); - server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } config_set_special_field("maxclients") { int orig_value = server.maxclients; @@ -1128,12 +828,6 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } config_set_special_field_with_alias("slave-announce-ip", - "replica-announce-ip") - { - zfree(server.slave_announce_ip); - server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - /* Boolean fields. * config_set_bool_field(name,var). */ } config_set_bool_field( @@ -1151,86 +845,8 @@ void configSetCommand(client *c) { #endif /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "tcp-keepalive",server.tcpkeepalive,0,INT_MAX) { - } config_set_numerical_field( - "maxmemory-samples",server.maxmemory_samples,1,INT_MAX) { - } config_set_numerical_field( - "lfu-log-factor",server.lfu_log_factor,0,INT_MAX) { - } config_set_numerical_field( - "lfu-decay-time",server.lfu_decay_time,0,INT_MAX) { - } config_set_numerical_field( - "timeout",server.maxidletime,0,INT_MAX) { - } config_set_numerical_field( - "active-defrag-threshold-lower",server.active_defrag_threshold_lower,0,1000) { - } config_set_numerical_field( - "active-defrag-threshold-upper",server.active_defrag_threshold_upper,0,1000) { - } config_set_memory_field( - "active-defrag-ignore-bytes",server.active_defrag_ignore_bytes) { - } config_set_numerical_field( - "active-defrag-cycle-min",server.active_defrag_cycle_min,1,99) { - } config_set_numerical_field( - "active-defrag-cycle-max",server.active_defrag_cycle_max,1,99) { - } config_set_numerical_field( - "active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,1,LONG_MAX) { - } config_set_numerical_field( - "active-expire-effort",server.active_expire_effort,1,10) { - } config_set_numerical_field( - "auto-aof-rewrite-percentage",server.aof_rewrite_perc,0,INT_MAX){ - } config_set_numerical_field( - "hash-max-ziplist-entries",server.hash_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "hash-max-ziplist-value",server.hash_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-bytes",server.stream_node_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { - } config_set_numerical_field( - "list-max-ziplist-size",server.list_max_ziplist_size,INT_MIN,INT_MAX) { - } config_set_numerical_field( - "list-compress-depth",server.list_compress_depth,0,INT_MAX) { - } config_set_numerical_field( - "set-max-intset-entries",server.set_max_intset_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-entries",server.zset_max_ziplist_entries,0,LONG_MAX) { - } config_set_numerical_field( - "zset-max-ziplist-value",server.zset_max_ziplist_value,0,LONG_MAX) { - } config_set_numerical_field( - "hll-sparse-max-bytes",server.hll_sparse_max_bytes,0,LONG_MAX) { - } config_set_numerical_field( - "lua-time-limit",server.lua_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "slowlog-log-slower-than",server.slowlog_log_slower_than,-1,LLONG_MAX) { - } config_set_numerical_field( - "slowlog-max-len",ll,0,LONG_MAX) { - /* Cast to unsigned. */ - server.slowlog_max_len = (unsigned long)ll; - } config_set_numerical_field( - "tracking-table-max-fill",server.tracking_table_max_fill,0,100) { - } config_set_numerical_field( - "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ - } config_set_numerical_field( - "repl-ping-slave-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-ping-replica-period",server.repl_ping_slave_period,1,INT_MAX) { - } config_set_numerical_field( - "repl-timeout",server.repl_timeout,1,INT_MAX) { } config_set_numerical_field( "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "repl-diskless-sync-delay",server.repl_diskless_sync_delay,0,INT_MAX) { - } config_set_numerical_field( - "slave-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "replica-priority",server.slave_priority,0,INT_MAX) { - } config_set_numerical_field( - "rdb-key-save-delay",server.rdb_key_save_delay,0,LLONG_MAX) { - } config_set_numerical_field( - "key-load-delay",server.key_load_delay,0,LLONG_MAX) { - } config_set_numerical_field( - "slave-announce-port",server.slave_announce_port,0,65535) { - } config_set_numerical_field( - "replica-announce-port",server.slave_announce_port,0,65535) { } config_set_numerical_field( "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { refreshGoodSlavesCount(); @@ -1243,18 +859,6 @@ void configSetCommand(client *c) { } config_set_numerical_field( "min-replicas-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { refreshGoodSlavesCount(); - } config_set_numerical_field( - "cluster-node-timeout",server.cluster_node_timeout,0,LLONG_MAX) { - } config_set_numerical_field( - "cluster-announce-port",server.cluster_announce_port,0,65535) { - } config_set_numerical_field( - "cluster-announce-bus-port",server.cluster_announce_bus_port,0,65535) { - } config_set_numerical_field( - "cluster-migration-barrier",server.cluster_migration_barrier,0,INT_MAX){ - } config_set_numerical_field( - "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { - } config_set_numerical_field( - "cluster-replica-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { } config_set_numerical_field( "hz",server.config_hz,0,INT_MAX) { /* Hz is more an hint from the user, so we accept values out of range @@ -1267,7 +871,6 @@ void configSetCommand(client *c) { enableWatchdog(ll); else disableWatchdog(); - /* Memory fields. * config_set_memory_field(name,var) */ } config_set_memory_field("maxmemory",server.maxmemory) { @@ -1277,25 +880,10 @@ void configSetCommand(client *c) { } freeMemoryIfNeededAndSafe(); } - } config_set_memory_field( - "proto-max-bulk-len",server.proto_max_bulk_len) { - } config_set_memory_field( - "client-query-buffer-limit",server.client_max_querybuf_len) { + } config_set_memory_field( + "client-query-buffer-limit",server.client_max_querybuf_len) { } config_set_memory_field("repl-backlog-size",ll) { resizeReplicationBacklog(ll); - } config_set_memory_field("auto-aof-rewrite-min-size",ll) { - server.aof_rewrite_min_size = ll; - - /* Enumeration fields. - * config_set_enum_field(name,var,enum_var) */ - } config_set_enum_field( - "loglevel",server.verbosity,loglevel_enum) { - } config_set_enum_field( - "maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum) { - } config_set_enum_field( - "appendfsync",server.aof_fsync,aof_fsync_enum) { - } config_set_enum_field( - "repl-diskless-load",server.repl_diskless_load,repl_diskless_load_enum) { #ifdef USE_OPENSSL /* TLS fields. */ } config_set_special_field("tls-cert-file") { @@ -1436,13 +1024,6 @@ badfmt: /* Bad format errors */ } \ } while(0); -#define config_get_enum_field(_name,_var,_enumvar) do { \ - if (stringmatch(pattern,_name,1)) { \ - addReplyBulkCString(c,_name); \ - addReplyBulkCString(c,configEnumGetNameOrUnknown(_enumvar,_var)); \ - matches++; \ - } \ -} while(0); void configGetCommand(client *c) { robj *o = c->argv[2]; @@ -1452,17 +1033,23 @@ void configGetCommand(client *c) { int matches = 0; serverAssertWithInfo(c,o,sdsEncodedObject(o)); + /* Iterate the configs that are standard */ + for (standardConfig *config = configs; config->name != NULL; config++) { + if (stringmatch(pattern,config->name,1)) { + addReplyBulkCString(c,config->name); + config->interface.get(c,config->data); + matches++; + } + if (config->alias && stringmatch(pattern,config->alias,1)) { + addReplyBulkCString(c,config->alias); + config->interface.get(c,config->data); + matches++; + } + } + /* String values */ config_get_string_field("dbfilename",server.rdb_filename); - config_get_string_field("masteruser",server.masteruser); - config_get_string_field("masterauth",server.masterauth); - config_get_string_field("cluster-announce-ip",server.cluster_announce_ip); - config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); - config_get_string_field("aclfile",server.acl_filename); - config_get_string_field("pidfile",server.pidfile); - config_get_string_field("slave-announce-ip",server.slave_announce_ip); - config_get_string_field("replica-announce-ip",server.slave_announce_ip); #ifdef USE_OPENSSL config_get_string_field("tls-cert-file",server.tls_ctx_config.cert_file); config_get_string_field("tls-key-file",server.tls_ctx_config.key_file); @@ -1476,110 +1063,25 @@ void configGetCommand(client *c) { /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); - config_get_numerical_field("proto-max-bulk-len",server.proto_max_bulk_len); config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len); - config_get_numerical_field("maxmemory-samples",server.maxmemory_samples); - config_get_numerical_field("lfu-log-factor",server.lfu_log_factor); - config_get_numerical_field("lfu-decay-time",server.lfu_decay_time); - config_get_numerical_field("timeout",server.maxidletime); - config_get_numerical_field("active-defrag-threshold-lower",server.active_defrag_threshold_lower); - config_get_numerical_field("active-defrag-threshold-upper",server.active_defrag_threshold_upper); - config_get_numerical_field("active-defrag-ignore-bytes",server.active_defrag_ignore_bytes); - config_get_numerical_field("active-defrag-cycle-min",server.active_defrag_cycle_min); - config_get_numerical_field("active-defrag-cycle-max",server.active_defrag_cycle_max); - config_get_numerical_field("active-defrag-max-scan-fields",server.active_defrag_max_scan_fields); - config_get_numerical_field("active-expire-effort",server.active_expire_effort); - config_get_numerical_field("auto-aof-rewrite-percentage", - server.aof_rewrite_perc); - config_get_numerical_field("auto-aof-rewrite-min-size", - server.aof_rewrite_min_size); - config_get_numerical_field("hash-max-ziplist-entries", - server.hash_max_ziplist_entries); - config_get_numerical_field("hash-max-ziplist-value", - server.hash_max_ziplist_value); - config_get_numerical_field("stream-node-max-bytes", - server.stream_node_max_bytes); - config_get_numerical_field("stream-node-max-entries", - server.stream_node_max_entries); - config_get_numerical_field("list-max-ziplist-size", - server.list_max_ziplist_size); - config_get_numerical_field("list-compress-depth", - server.list_compress_depth); - config_get_numerical_field("set-max-intset-entries", - server.set_max_intset_entries); - config_get_numerical_field("zset-max-ziplist-entries", - server.zset_max_ziplist_entries); - config_get_numerical_field("zset-max-ziplist-value", - server.zset_max_ziplist_value); - config_get_numerical_field("hll-sparse-max-bytes", - server.hll_sparse_max_bytes); - config_get_numerical_field("lua-time-limit",server.lua_time_limit); - config_get_numerical_field("slowlog-log-slower-than", - server.slowlog_log_slower_than); - config_get_numerical_field("latency-monitor-threshold", - server.latency_monitor_threshold); - config_get_numerical_field("slowlog-max-len", server.slowlog_max_len); - config_get_numerical_field("tracking-table-max-fill", server.tracking_table_max_fill); - config_get_numerical_field("port",server.port); config_get_numerical_field("tls-port",server.tls_port); - config_get_numerical_field("cluster-announce-port",server.cluster_announce_port); - config_get_numerical_field("cluster-announce-bus-port",server.cluster_announce_bus_port); - config_get_numerical_field("tcp-backlog",server.tcp_backlog); - config_get_numerical_field("databases",server.dbnum); - config_get_numerical_field("io-threads",server.io_threads_num); - config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-ping-replica-period",server.repl_ping_slave_period); - config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); - config_get_numerical_field("slave-priority",server.slave_priority); - config_get_numerical_field("replica-priority",server.slave_priority); - config_get_numerical_field("slave-announce-port",server.slave_announce_port); - config_get_numerical_field("replica-announce-port",server.slave_announce_port); config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); config_get_numerical_field("hz",server.config_hz); - config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); - config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); - config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); - config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); - config_get_numerical_field("rdb-key-save-delay",server.rdb_key_save_delay); - config_get_numerical_field("key-load-delay",server.key_load_delay); - config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); /* Bool (yes/no) values */ - /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { - config_get_bool_field(config->name, *(config->config)); - if (config->alias) { - config_get_bool_field(config->alias, *(config->config)); - } - } - config_get_bool_field("activedefrag", server.active_defrag_enabled); config_get_bool_field("tls-cluster",server.tls_cluster); config_get_bool_field("tls-replication",server.tls_replication); config_get_bool_field("tls-auth-clients",server.tls_auth_clients); config_get_bool_field("tls-prefer-server-ciphers", server.tls_ctx_config.prefer_server_ciphers); - /* Enum values */ - config_get_enum_field("maxmemory-policy", - server.maxmemory_policy,maxmemory_policy_enum); - config_get_enum_field("loglevel", - server.verbosity,loglevel_enum); - config_get_enum_field("supervised", - server.supervised_mode,supervised_mode_enum); - config_get_enum_field("appendfsync", - server.aof_fsync,aof_fsync_enum); - config_get_enum_field("syslog-facility", - server.syslog_facility,syslog_facility_enum); - config_get_enum_field("repl-diskless-load", - server.repl_diskless_load,repl_diskless_load_enum); /* Everything we can't handle with macros follows. */ @@ -1900,7 +1402,7 @@ int rewriteConfigFormatMemory(char *buf, size_t len, long long bytes) { } /* Rewrite a simple "option-name " configuration option. */ -void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigBytesOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { char buf[64]; int force = value != defvalue; sds line; @@ -1920,7 +1422,7 @@ void rewriteConfigYesNoOption(struct rewriteConfigState *state, const char *opti } /* Rewrite a string option. */ -void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { +void rewriteConfigStringOption(struct rewriteConfigState *state, const char *option, char *value, const char *defvalue) { int force = 1; sds line; @@ -1942,7 +1444,7 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, c } /* Rewrite a numerical (long long range) option. */ -void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { +void rewriteConfigNumericalOption(struct rewriteConfigState *state, const char *option, long long value, long long defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); @@ -1960,7 +1462,7 @@ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, in /* Rewrite an enumeration option. It takes as usually state and option name, * and in addition the enumeration array and the default value for the * option. */ -void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, configEnum *ce, int defval) { +void rewriteConfigEnumOption(struct rewriteConfigState *state, const char *option, int value, configEnum *ce, int defval) { sds line; const char *name = configEnumGetNameOrUnknown(ce,value); int force = value != defval; @@ -1969,18 +1471,6 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int rewriteConfigRewriteLine(state,option,line,force); } -/* Rewrite the syslog-facility option. */ -void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { - int value = server.syslog_facility; - int force = value != LOG_LOCAL0; - const char *name = NULL, *option = "syslog-facility"; - sds line; - - name = configEnumGetNameOrUnknown(syslog_facility_enum,value); - line = sdscatprintf(sdsempty(),"%s %s",option,name); - rewriteConfigRewriteLine(state,option,line,force); -} - /* Rewrite the save option. */ void rewriteConfigSaveOption(struct rewriteConfigState *state) { int j; @@ -2282,95 +1772,35 @@ int rewriteConfig(char *path) { * the rewrite state. */ /* Iterate the configs that are standard */ - for (configYesNo *config = configs_yesno; config->name != NULL; config++) { - rewriteConfigYesNoOption(state,config->name,*(config->config),config->default_value); + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.rewrite(config->data, config->name, state); } - rewriteConfigStringOption(state,"pidfile",server.pidfile,CONFIG_DEFAULT_PID_FILE); - rewriteConfigNumericalOption(state,"tls-port",server.tls_port,CONFIG_DEFAULT_SERVER_TLS_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-port",server.cluster_announce_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT); - rewriteConfigNumericalOption(state,"cluster-announce-bus-port",server.cluster_announce_bus_port,CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT); - rewriteConfigNumericalOption(state,"tcp-backlog",server.tcp_backlog,CONFIG_DEFAULT_TCP_BACKLOG); rewriteConfigBindOption(state); - rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); - rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); - rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"replica-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); - rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); - rewriteConfigStringOption(state,"aclfile",server.acl_filename,CONFIG_DEFAULT_ACL_FILENAME); rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,CONFIG_DEFAULT_SYSLOG_IDENT); - rewriteConfigSyslogfacilityOption(state); rewriteConfigSaveOption(state); rewriteConfigUserOption(state); - rewriteConfigNumericalOption(state,"databases",server.dbnum,CONFIG_DEFAULT_DBNUM); - rewriteConfigNumericalOption(state,"io-threads",server.dbnum,CONFIG_DEFAULT_IO_THREADS_NUM); rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigStringOption(state,"replica-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); - rewriteConfigStringOption(state,"masteruser",server.masteruser,NULL); - rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); - rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); - rewriteConfigNumericalOption(state,"repl-ping-replica-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); - rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); - rewriteConfigEnumOption(state,"repl-diskless-load",server.repl_diskless_load,repl_diskless_load_enum,CONFIG_DEFAULT_REPL_DISKLESS_LOAD); - rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"replica-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-replicas-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-replicas-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); rewriteConfigRequirepassOption(state,"requirepass"); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); - rewriteConfigBytesOption(state,"proto-max-bulk-len",server.proto_max_bulk_len,CONFIG_DEFAULT_PROTO_MAX_BULK_LEN); rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); - rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy,maxmemory_policy_enum,CONFIG_DEFAULT_MAXMEMORY_POLICY); - rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,CONFIG_DEFAULT_MAXMEMORY_SAMPLES); - rewriteConfigNumericalOption(state,"lfu-log-factor",server.lfu_log_factor,CONFIG_DEFAULT_LFU_LOG_FACTOR); - rewriteConfigNumericalOption(state,"lfu-decay-time",server.lfu_decay_time,CONFIG_DEFAULT_LFU_DECAY_TIME); - rewriteConfigNumericalOption(state,"active-defrag-threshold-lower",server.active_defrag_threshold_lower,CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER); - rewriteConfigNumericalOption(state,"active-defrag-threshold-upper",server.active_defrag_threshold_upper,CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER); - rewriteConfigBytesOption(state,"active-defrag-ignore-bytes",server.active_defrag_ignore_bytes,CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES); - rewriteConfigNumericalOption(state,"active-defrag-cycle-min",server.active_defrag_cycle_min,CONFIG_DEFAULT_DEFRAG_CYCLE_MIN); - rewriteConfigNumericalOption(state,"active-defrag-cycle-max",server.active_defrag_cycle_max,CONFIG_DEFAULT_DEFRAG_CYCLE_MAX); - rewriteConfigNumericalOption(state,"active-defrag-max-scan-fields",server.active_defrag_max_scan_fields,CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS); - rewriteConfigNumericalOption(state,"active-expire-effort",server.active_expire_effort,CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT); rewriteConfigYesNoOption(state,"appendonly",server.aof_enabled,0); rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync,aof_fsync_enum,CONFIG_DEFAULT_AOF_FSYNC); - rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,AOF_REWRITE_PERC); - rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); - rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,LUA_SCRIPT_TIME_LIMIT); rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); - rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); - rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); - rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); - rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); - rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); - rewriteConfigNumericalOption(state,"tracking-table-max-fill",server.tracking_table_max_fill,CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL); rewriteConfigNotifykeyspaceeventsOption(state); - rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,OBJ_HASH_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,OBJ_HASH_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"stream-node-max-bytes",server.stream_node_max_bytes,OBJ_STREAM_NODE_MAX_BYTES); - rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); - rewriteConfigNumericalOption(state,"list-max-ziplist-size",server.list_max_ziplist_size,OBJ_LIST_MAX_ZIPLIST_SIZE); - rewriteConfigNumericalOption(state,"list-compress-depth",server.list_compress_depth,OBJ_LIST_COMPRESS_DEPTH); - rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,OBJ_SET_MAX_INTSET_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,OBJ_ZSET_MAX_ZIPLIST_ENTRIES); - rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,OBJ_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigNumericalOption(state,"hll-sparse-max-bytes",server.hll_sparse_max_bytes,CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES); rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); - rewriteConfigEnumOption(state,"supervised",server.supervised_mode,supervised_mode_enum,SUPERVISED_NONE); - rewriteConfigNumericalOption(state,"rdb-key-save-delay",server.rdb_key_save_delay,CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY); - rewriteConfigNumericalOption(state,"key-load-delay",server.key_load_delay,CONFIG_DEFAULT_KEY_LOAD_DELAY); + #ifdef USE_OPENSSL rewriteConfigYesNoOption(state,"tls-cluster",server.tls_cluster,0); rewriteConfigYesNoOption(state,"tls-replication",server.tls_replication,0); @@ -2404,6 +1834,422 @@ int rewriteConfig(char *path) { return retval; } +/*----------------------------------------------------------------------------- + * Configs that fit one of the major types and require no special handling + *----------------------------------------------------------------------------*/ +#define LOADBUF_SIZE 256 +static char loadbuf[LOADBUF_SIZE]; + +#define MEMORY_CONFIG 1 +#define INTEGER_CONFIG 0 + +#define MODIFIABLE_CONFIG 1 +#define IMMUTABLE_CONFIG 0 + +#define embedCommonConfig(config_name, config_alias, is_modifiable) \ + .name = (config_name), \ + .alias = (config_alias), \ + .modifiable = (is_modifiable), + +#define embedConfigInterface(loadfn, setfn, getfn, rewritefn) .interface = { \ + .load = (loadfn), \ + .set = (setfn), \ + .get = (getfn), \ + .rewrite = (rewritefn) \ +}, + +/* + * What follows is the generic config types that are supported. To add a new + * config with one of these types, add it to the standardConfig table with + * the creation macro for each type. + * + * Each type contains the following: + * * A function defining how to load this type on startup. + * * A function defining how to update this type on CONFIG SET. + * * A function defining how to serialize this type on CONFIG SET. + * * A function defining how to rewrite this type on CONFIG REWRITE. + * * A Macro defining how to create this type. + */ +/* Bool Configs */ +static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + if ((*(data.yesno.config) = yesnotoi(argv[1])) == -1) { + *err = "argument must be 'yes' or 'no'"; + return 0; + } + return 1; +} + +static int boolConfigSet(typeData data, sds value) { + int yn = yesnotoi(value); + if (yn == -1) return 0; + *(data.yesno.config) = yn; + return 1; +} + +static void boolConfigGet(client *c, typeData data) { + addReplyBulkCString(c, data.yesno.config ? "yes" : "no"); +} + +static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); +} + +#define createBoolConfig(name, alias, modifiable, config_addr, default) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + .data.yesno = { \ + .config = &(config_addr), \ + .default_value = (default) \ + } \ +} + +/* String Configs */ +static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + zfree(*data.string.config); + *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; + return 1; +} + +static int stringConfigSet(typeData data, sds value) { + zfree(*data.string.config); + *data.string.config = value ? zstrdup(value) : NULL; + return 1; +} + +static void stringConfigGet(client *c, typeData data) { + addReplyBulkCString(c, *data.string.config ? *data.string.config : ""); +} + +static void stringConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); +} + +#define createStringConfig(name, alias, modifiable, config_addr, default) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + .data.string = { \ + .config = &(config_addr), \ + .default_value = (default), \ + } \ +} + +/* Enum configs */ +static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + + int enumval = configEnumGetValue(data.enumd.enum_value, argv[1]); + if (enumval == INT_MIN) { + sds enumerr = sdsnew("argument must be one of the following: "); + configEnum *enumNode = data.enumd.enum_value; + while(enumNode->name != NULL) { + sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); + sdscatlen(enumerr, ", ", 2); + enumNode++; + } + + enumerr[sdslen(enumerr) - 2] = '\0'; + + /* Make sure we don't overrun the fixed buffer */ + enumerr[LOADBUF_SIZE - 1] = '\0'; + strncpy(loadbuf, enumerr, LOADBUF_SIZE); + + sdsfree(enumerr); + *err = loadbuf; + return 0; + } + *(data.enumd.config) = enumval; + return 1; +} + +static int configEnumSet(typeData data, sds value) { + int enumval = configEnumGetValue(data.enumd.enum_value, value); + if (enumval == INT_MIN) return 0; + *(data.enumd.config) = enumval; + return 1; +} + +static void configEnumGet(client *c, typeData data) { + addReplyBulkCString(c, configEnumGetNameOrUnknown(data.enumd.enum_value,*data.enumd.config)); +} + +static void configEnumRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); +} + +#define createEnumConfig(name, alias, modifiable, enum, config_addr, default) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ + .data.enumd = { \ + .config = &(config_addr), \ + .default_value = (default), \ + .enum_value = (enum) \ + } \ +} + +/* Numeric configs */ +static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { + long long ll; + + if (argc != 2) { + *err = "wrong number of arguments"; + return 0; + } + + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(argv[1], &memerr); + if (memerr || ll < 0) { + *err = "argument must be a memory value"; + return 0; + } + } else { + if (!string2ll(argv[1], sdslen(argv[1]),&ll)) { + *err = "argument couldn't be parsed into an integer" ; + return 0; + } + } + + if (ll > data.numeric.upper_bound || + ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } + + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { + *(data.numeric.config.i) = (int) ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { + *(data.numeric.config.ul) = (unsigned long) ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { + *(data.numeric.config.ll) = ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + *(data.numeric.config.st) = (size_t) ll; + } + + return 1; +} + +static int numericConfigSet(typeData data, sds value) { + long long ll; + if (data.numeric.is_memory) { + int memerr; + ll = memtoll(value, &memerr); + if (memerr || ll < 0) return 0; + } else { + if (!string2ll(value, sdslen(value),&ll)) return 0; + } + + if (ll > data.numeric.upper_bound || + ll < data.numeric.lower_bound) { + return 0; + } + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { + *(data.numeric.config.i) = (int) ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { + *(data.numeric.config.ul) = (unsigned long) ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { + *(data.numeric.config.ll) = ll; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + *(data.numeric.config.st) = (size_t) ll; + } + + return 1; +} + +static void numericConfigGet(client *c, typeData data) { + char buf[128]; + long long value = 0; + + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { + value = *(data.numeric.config.i); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { + value = *(data.numeric.config.ul); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { + value = *(data.numeric.config.ll); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + value = *(data.numeric.config.st); + } + + ll2string(buf, sizeof(buf), value); + addReplyBulkCString(c, buf); +} + +static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { + long long value; + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { + value = *(data.numeric.config.i); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { + value = *(data.numeric.config.ul); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { + value = *(data.numeric.config.ll); + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + value = *(data.numeric.config.st); + } + + if (data.numeric.is_memory) { + rewriteConfigBytesOption(state, name, value, data.numeric.default_value); + } else { + rewriteConfigNumericalOption(state, name, value, data.numeric.default_value); + } +} + +#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) { \ + embedCommonConfig(name, alias, modifiable) \ + embedConfigInterface(numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + .data.numeric = { \ + .lower_bound = (lower), \ + .upper_bound = (upper), \ + .default_value = (default), \ + .is_memory = (memory), + +#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + .numeric_type = NUMERIC_TYPE_INT, \ + .config.i = &(config_addr) \ + } \ +} + +#define createUnsignedLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + .numeric_type = NUMERIC_TYPE_UNSIGNED_LONG, \ + .config.ul = &(config_addr) \ + } \ +} + +#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + .numeric_type = NUMERIC_TYPE_LONG_LONG, \ + .config.ll = &(config_addr) \ + } \ +} + +#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ + .numeric_type = NUMERIC_TYPE_SIZE_T, \ + .config.st = &(config_addr) \ + } \ +} + +standardConfig configs[] = { + /* Bool configs */ + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, CONFIG_DEFAULT_IO_THREADS_DO_READS), + createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ), + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER), + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1), + + /* String Configs */ + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, server.unixsocket, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, server.pidfile, CONFIG_DEFAULT_PID_FILE), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, server.masteruser, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, server.masterauth, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, server.cluster_announce_ip, NULL), + + /* Enum Configs */ + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC), + + /* Integer configs */ + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG), + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG), + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG), + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG), + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG), + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG), + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG), + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG), + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG), + createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG), + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG), + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG), + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG), + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG), + + /* Unsigned Long configs */ + createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG), + createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG), + + /* Long Long configs */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, OBJ_STREAM_NODE_MAX_ENTRIES, INTEGER_CONFIG), + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG), + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG), + createLongLongConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.aof_rewrite_min_size, AOF_REWRITE_MIN_SIZE, MEMORY_CONFIG), + + /* Size_t configs */ + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG), + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG), + + /* NULL Terminator */ + {NULL} +}; + /*----------------------------------------------------------------------------- * CONFIG command entry point *----------------------------------------------------------------------------*/ From 40e3cfb15339e62bb6f9bbc2f06bd2dcab6284c9 Mon Sep 17 00:00:00 2001 From: Daniel Dai <764122422@qq.com> Date: Wed, 20 Nov 2019 20:18:00 -0500 Subject: [PATCH 296/320] macro fix macro fix --- src/bio.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bio.h b/src/bio.h index 4b15d1c4d..6c2155941 100644 --- a/src/bio.h +++ b/src/bio.h @@ -27,6 +27,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef __BIO_H +#define __BIO_H + /* Exported API */ void bioInit(void); void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3); @@ -40,3 +43,5 @@ void bioKillThreads(void); #define BIO_AOF_FSYNC 1 /* Deferred AOF fsync. */ #define BIO_LAZY_FREE 2 /* Deferred objects freeing. */ #define BIO_NUM_OPS 3 + +#endif From d97625dcf6c871ef2005144941b5cf6055f50d03 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2019 10:01:49 +0100 Subject: [PATCH 297/320] Recomment PR #6346. --- src/module.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/module.c b/src/module.c index 77b885b00..4c609e4ff 100644 --- a/src/module.c +++ b/src/module.c @@ -5995,19 +5995,23 @@ int RM_CommandFilterArgDelete(RedisModuleCommandFilterCtx *fctx, int pos) return REDISMODULE_OK; } -/* - * For a given pointer, return The amount of memory - * allocated for this pointer. +/* For a given pointer allocated via RedisModule_Alloc() or + * RedisModule_Realloc(), return the amount of memory allocated for it. + * Note that this may be different (larger) than the memory we allocated + * with the allocation calls, since sometimes the underlying allocator + * will allocate more memory. */ size_t RM_MallocSize(void* ptr){ return zmalloc_size(ptr); } -/* - * Return the a number between 0 to 1 indicating - * the amount of memory currently used. - * 0 - no memory limit - * 1 and above, memory limit reached. +/* Return the a number between 0 to 1 indicating the amount of memory + * currently used, relative to the Redis "maxmemory" configuration. + * + * 0 - No memory limit configured. + * Between 0 and 1 - The percentage of the memory used normalized in 0-1 range. + * Exactly 1 - Memory limit reached. + * Greater 1 - More memory used than the configured limit. */ float RM_GetUsedMemoryRatio(){ float level; From d57a2b9ef229e519173492bcd5ce888fe8fc6a8f Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Thu, 21 Nov 2019 21:31:53 -0800 Subject: [PATCH 298/320] Added a flag for strings that are stored as NULL --- src/config.c | 56 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/config.c b/src/config.c index f6440272e..b8df1f1c3 100644 --- a/src/config.c +++ b/src/config.c @@ -115,6 +115,9 @@ typedef struct boolConfigData { typedef struct stringConfigData { char **config; /* The pointer to the server config this value is stored in */ const char *default_value; /* The default value of the config on rewrite */ + int convert_empty_to_null; /* A boolean indicating if empty strings should + be stored as a NULL value. */ + } stringConfigData; typedef struct enumConfigData { @@ -145,7 +148,7 @@ typedef struct numericConfigData { } numericConfigData; typedef union typeData { - boolConfigData yesno; + boolConfigData bool; stringConfigData string; enumConfigData enumd; numericConfigData numeric; @@ -166,7 +169,7 @@ typedef struct standardConfig { const char *name; /* The user visible name of this config */ const char *alias; /* An alias that can also be used for this config */ const int modifiable; /* Can this value be updated by CONFIG SET? */ - typeInterface interface; /* The function points that define the type interface */ + typeInterface interface; /* The function pointers that define the type interface */ typeData data; /* The type specific data exposed used by the interface */ } standardConfig; @@ -1840,9 +1843,6 @@ int rewriteConfig(char *path) { #define LOADBUF_SIZE 256 static char loadbuf[LOADBUF_SIZE]; -#define MEMORY_CONFIG 1 -#define INTEGER_CONFIG 0 - #define MODIFIABLE_CONFIG 1 #define IMMUTABLE_CONFIG 0 @@ -1876,7 +1876,7 @@ static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { *err = "wrong number of arguments"; return 0; } - if ((*(data.yesno.config) = yesnotoi(argv[1])) == -1) { + if ((*(data.bool.config) = yesnotoi(argv[1])) == -1) { *err = "argument must be 'yes' or 'no'"; return 0; } @@ -1886,22 +1886,22 @@ static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { static int boolConfigSet(typeData data, sds value) { int yn = yesnotoi(value); if (yn == -1) return 0; - *(data.yesno.config) = yn; + *(data.bool.config) = yn; return 1; } static void boolConfigGet(client *c, typeData data) { - addReplyBulkCString(c, data.yesno.config ? "yes" : "no"); + addReplyBulkCString(c, data.bool.config ? "yes" : "no"); } static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { - rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); + rewriteConfigYesNoOption(state, name,*(data.bool.config), data.bool.default_value); } #define createBoolConfig(name, alias, modifiable, config_addr, default) { \ embedCommonConfig(name, alias, modifiable) \ embedConfigInterface(boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ - .data.yesno = { \ + .data.bool = { \ .config = &(config_addr), \ .default_value = (default) \ } \ @@ -1914,13 +1914,21 @@ static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { return 0; } zfree(*data.string.config); - *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; + if (data.string.convert_empty_to_null) { + *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; + } else { + *data.string.config = zstrdup(argv[1]); + } return 1; } static int stringConfigSet(typeData data, sds value) { zfree(*data.string.config); - *data.string.config = value ? zstrdup(value) : NULL; + if (data.string.convert_empty_to_null) { + *data.string.config = value[0] ? zstrdup(value) : NULL; + } else { + *data.string.config = zstrdup(value); + } return 1; } @@ -1932,12 +1940,16 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC rewriteConfigStringOption(state, name,*(data.string.config), data.string.default_value); } -#define createStringConfig(name, alias, modifiable, config_addr, default) { \ +#define ALLOW_EMPTY_STRING 0 +#define EMPTY_STRING_IS_NULL 1 + +#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default) { \ embedCommonConfig(name, alias, modifiable) \ embedConfigInterface(stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ .data.string = { \ .config = &(config_addr), \ .default_value = (default), \ + .convert_empty_to_null = (empty_to_null) \ } \ } @@ -2057,6 +2069,7 @@ static int numericConfigSet(typeData data, sds value) { ll < data.numeric.lower_bound) { return 0; } + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { *(data.numeric.config.i) = (int) ll; } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { @@ -2107,6 +2120,9 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite } } +#define INTEGER_CONFIG 0 +#define MEMORY_CONFIG 1 + #define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) { \ embedCommonConfig(name, alias, modifiable) \ embedConfigInterface(numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ @@ -2176,13 +2192,13 @@ standardConfig configs[] = { createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1), /* String Configs */ - createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), - createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, server.unixsocket, NULL), - createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, server.pidfile, CONFIG_DEFAULT_PID_FILE), - createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, server.masteruser, NULL), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, server.masterauth, NULL), - createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, server.cluster_announce_ip, NULL), + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE), From 1f35b8ce05bdb92e5c53622d353291710d4b8447 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 22 Nov 2019 05:49:52 +0000 Subject: [PATCH 299/320] Fixed some linux warnings --- src/config.c | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/config.c b/src/config.c index b8df1f1c3..d9f8fb4c3 100644 --- a/src/config.c +++ b/src/config.c @@ -148,7 +148,7 @@ typedef struct numericConfigData { } numericConfigData; typedef union typeData { - boolConfigData bool; + boolConfigData yesno; stringConfigData string; enumConfigData enumd; numericConfigData numeric; @@ -409,6 +409,10 @@ void loadServerConfigFromString(char *config) { } zfree(server.aof_filename); server.aof_filename = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && + argc == 2) + { + server.aof_rewrite_min_size = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; @@ -427,6 +431,8 @@ void loadServerConfigFromString(char *config) { } zfree(server.rdb_filename); server.rdb_filename = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { + server.stream_node_max_entries = atoi(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -850,6 +856,8 @@ void configSetCommand(client *c) { * config_set_numerical_field(name,var,min,max) */ } config_set_numerical_field( "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { + } config_set_numerical_field( + "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { } config_set_numerical_field( "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { refreshGoodSlavesCount(); @@ -887,6 +895,8 @@ void configSetCommand(client *c) { "client-query-buffer-limit",server.client_max_querybuf_len) { } config_set_memory_field("repl-backlog-size",ll) { resizeReplicationBacklog(ll); + } config_set_memory_field("auto-aof-rewrite-min-size",ll) { + server.aof_rewrite_min_size = ll; #ifdef USE_OPENSSL /* TLS fields. */ } config_set_special_field("tls-cert-file") { @@ -1067,6 +1077,10 @@ void configGetCommand(client *c) { /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len); + config_get_numerical_field("auto-aof-rewrite-min-size", + server.aof_rewrite_min_size); + config_get_numerical_field("stream-node-max-entries", + server.stream_node_max_entries); config_get_numerical_field("tls-port",server.tls_port); config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); @@ -1798,8 +1812,10 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"appendonly",server.aof_enabled,0); rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); + rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); + rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); @@ -1876,7 +1892,7 @@ static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { *err = "wrong number of arguments"; return 0; } - if ((*(data.bool.config) = yesnotoi(argv[1])) == -1) { + if ((*(data.yesno.config) = yesnotoi(argv[1])) == -1) { *err = "argument must be 'yes' or 'no'"; return 0; } @@ -1886,22 +1902,22 @@ static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { static int boolConfigSet(typeData data, sds value) { int yn = yesnotoi(value); if (yn == -1) return 0; - *(data.bool.config) = yn; + *(data.yesno.config) = yn; return 1; } static void boolConfigGet(client *c, typeData data) { - addReplyBulkCString(c, data.bool.config ? "yes" : "no"); + addReplyBulkCString(c, data.yesno.config ? "yes" : "no"); } static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { - rewriteConfigYesNoOption(state, name,*(data.bool.config), data.bool.default_value); + rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); } #define createBoolConfig(name, alias, modifiable, config_addr, default) { \ embedCommonConfig(name, alias, modifiable) \ embedConfigInterface(boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ - .data.bool = { \ + .data.yesno = { \ .config = &(config_addr), \ .default_value = (default) \ } \ @@ -2102,7 +2118,7 @@ static void numericConfigGet(client *c, typeData data) { } static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { - long long value; + long long value = 0; if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { value = *(data.numeric.config.i); } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { @@ -2244,13 +2260,11 @@ standardConfig configs[] = { createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG), /* Long Long configs */ - createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.stream_node_max_entries, OBJ_STREAM_NODE_MAX_ENTRIES, INTEGER_CONFIG), createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG), createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG), createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG), createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG), createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG), - createLongLongConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.aof_rewrite_min_size, AOF_REWRITE_MIN_SIZE, MEMORY_CONFIG), /* Size_t configs */ createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), From c0891d5cca65e2076be36b0b926fb0e130e80db5 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 22 Nov 2019 06:28:50 +0000 Subject: [PATCH 300/320] Fixed a bug with enum log printing --- src/config.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/config.c b/src/config.c index d9f8fb4c3..01fec7ff7 100644 --- a/src/config.c +++ b/src/config.c @@ -662,8 +662,8 @@ void loadServerConfig(char *filename, char *options) { #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { -#define config_set_special_field_with_alias(_name,_name2) \ - } else if (!strcasecmp(c->argv[2]->ptr,_name) || \ +#define config_set_special_field_with_alias(_name1,_name2) \ + } else if (!strcasecmp(c->argv[2]->ptr,_name1) || \ !strcasecmp(c->argv[2]->ptr,_name2)) { #define config_set_else } else @@ -854,10 +854,10 @@ void configSetCommand(client *c) { #endif /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { } config_set_numerical_field( "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { + } config_set_numerical_field( + "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { } config_set_numerical_field( "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { refreshGoodSlavesCount(); @@ -891,8 +891,8 @@ void configSetCommand(client *c) { } freeMemoryIfNeededAndSafe(); } - } config_set_memory_field( - "client-query-buffer-limit",server.client_max_querybuf_len) { + } config_set_memory_field( + "client-query-buffer-limit",server.client_max_querybuf_len) { } config_set_memory_field("repl-backlog-size",ll) { resizeReplicationBacklog(ll); } config_set_memory_field("auto-aof-rewrite-min-size",ll) { @@ -1811,8 +1811,8 @@ int rewriteConfig(char *path) { rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); rewriteConfigYesNoOption(state,"appendonly",server.aof_enabled,0); rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); + rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); @@ -1981,8 +1981,8 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { sds enumerr = sdsnew("argument must be one of the following: "); configEnum *enumNode = data.enumd.enum_value; while(enumNode->name != NULL) { - sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); - sdscatlen(enumerr, ", ", 2); + enumerr = sdscatlen(enumerr, enumNode->name, strlen(enumNode->name)); + enumerr = sdscatlen(enumerr, ", ", 2); enumNode++; } From 678cf14f4a1cbfba984f7bf5c149f2b1612e6f71 Mon Sep 17 00:00:00 2001 From: Madelyn Olson Date: Fri, 22 Nov 2019 06:44:12 +0000 Subject: [PATCH 301/320] Changed a tab to a space --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 01fec7ff7..550f9e0f8 100644 --- a/src/config.c +++ b/src/config.c @@ -363,7 +363,7 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { server.maxmemory = memtoll(argv[1],NULL); - } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { + } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { server.client_max_querybuf_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"slaveof") || !strcasecmp(argv[0],"replicaof")) && argc == 3) { From b7d9ef6033fe9ecdbcda9efa734932071bc05b88 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 22 Nov 2019 17:54:22 +0100 Subject: [PATCH 302/320] config.c: remove trailing spaces, trim comments to 80 cols. --- src/config.c | 205 +++++++++++++++++++++++++-------------------------- 1 file changed, 102 insertions(+), 103 deletions(-) diff --git a/src/config.c b/src/config.c index 550f9e0f8..77dd94d62 100644 --- a/src/config.c +++ b/src/config.c @@ -105,7 +105,7 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {1024*1024*32, 1024*1024*8, 60} /* pubsub */ }; -/* Configuration values that require no special handling to set, get, load or +/* Configuration values that require no special handling to set, get, load or * rewrite. */ typedef struct boolConfigData { int *config; /* The pointer to the server config this value is stored in */ @@ -113,11 +113,10 @@ typedef struct boolConfigData { } boolConfigData; typedef struct stringConfigData { - char **config; /* The pointer to the server config this value is stored in */ - const char *default_value; /* The default value of the config on rewrite */ - int convert_empty_to_null; /* A boolean indicating if empty strings should + char **config; /* Pointer to the server config this value is stored in. */ + const char *default_value; /* Default value of the config on rewrite. */ + int convert_empty_to_null; /* Boolean indicating if empty strings should be stored as a NULL value. */ - } stringConfigData; typedef struct enumConfigData { @@ -127,9 +126,9 @@ typedef struct enumConfigData { } enumConfigData; typedef enum numericType { - NUMERIC_TYPE_INT, - NUMERIC_TYPE_LONG_LONG, - NUMERIC_TYPE_UNSIGNED_LONG, + NUMERIC_TYPE_INT, + NUMERIC_TYPE_LONG_LONG, + NUMERIC_TYPE_UNSIGNED_LONG, NUMERIC_TYPE_SIZE_T } numericType; @@ -137,7 +136,7 @@ typedef struct numericConfigData { union { int *i; long long *ll; - unsigned long *ul; + unsigned long *ul; size_t *st; } config; /* The pointer to the numeric config this value is stored in */ int is_memory; /* Indicates if this value can be loaded as a memory value */ @@ -156,9 +155,9 @@ typedef union typeData { typedef struct typeInterface { /* Called on server start, should return 1 on success, 0 on error and should set err */ - int (*load)(typeData data, sds *argc, int argv, char **err); + int (*load)(typeData data, sds *argc, int argv, char **err); /* Called on CONFIG SET, returns 1 on success, 0 on error */ - int (*set)(typeData data, sds value); + int (*set)(typeData data, sds value); /* Called on CONFIG GET, required to add output to the client */ void (*get)(client *c, typeData data); /* Called on CONFIG REWRITE, required to rewrite the config state */ @@ -282,7 +281,7 @@ void loadServerConfigFromString(char *config) { int match = 0; for (standardConfig *config = configs; config->name != NULL; config++) { if ((!strcasecmp(argv[0],config->name) || - (config->alias && !strcasecmp(argv[0],config->alias)))) + (config->alias && !strcasecmp(argv[0],config->alias)))) { if (!config->interface.load(config->data, argv, argc, &err)) { goto loaderr; @@ -679,7 +678,7 @@ void configSetCommand(client *c) { /* Iterate the configs that are standard */ for (standardConfig *config = configs; config->name != NULL; config++) { if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || - (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) + (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { if (!config->interface.set(config->data,o->ptr)) { goto badfmt; @@ -1874,18 +1873,18 @@ static char loadbuf[LOADBUF_SIZE]; .rewrite = (rewritefn) \ }, -/* - * What follows is the generic config types that are supported. To add a new +/* What follows is the generic config types that are supported. To add a new * config with one of these types, add it to the standardConfig table with * the creation macro for each type. - * + * * Each type contains the following: * * A function defining how to load this type on startup. * * A function defining how to update this type on CONFIG SET. * * A function defining how to serialize this type on CONFIG SET. * * A function defining how to rewrite this type on CONFIG REWRITE. - * * A Macro defining how to create this type. + * * A Macro defining how to create this type. */ + /* Bool Configs */ static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { if (argc != 2) { @@ -2040,19 +2039,19 @@ static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { if (memerr || ll < 0) { *err = "argument must be a memory value"; return 0; - } + } } else { if (!string2ll(argv[1], sdslen(argv[1]),&ll)) { *err = "argument couldn't be parsed into an integer" ; - return 0; + return 0; } } if (ll > data.numeric.upper_bound || ll < data.numeric.lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %lld and %lld inclusive", - data.numeric.lower_bound, + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, data.numeric.upper_bound); *err = loadbuf; return 0; @@ -2178,103 +2177,103 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite standardConfig configs[] = { /* Bool configs */ - createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM), + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM), createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0), createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, CONFIG_DEFAULT_IO_THREADS_DO_READS), createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1), - createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO), - createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE), - createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION), - createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING), - createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR), - createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ), - createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION), - createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE), - createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL), - createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY), - createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC), - createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED), - createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC), - createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE), - createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE), - createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC), - createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED), - createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE), - createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER), - createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH), - createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA), - createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY), - createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY), - createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ), + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER), + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1), /* String Configs */ - createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), - createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL), - createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE), - createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL), - createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL), + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL), /* Enum Configs */ - createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE), - createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0), - createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD), - createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY), - createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY), - createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC), + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC), /* Integer configs */ - createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG), - createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG), - createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG), - createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG), - createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG), - createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG), - createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG), - createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG), - createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG), - createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG), - createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG), - createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG), - createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG), - createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG), - createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG), - createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG), - createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG), - createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG), - createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG), - createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG), - createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG), - createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG), - createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG), - createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG), + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG), + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG), + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG), + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG), + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG), + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG), + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG), + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG), + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG), + createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG), + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG), + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG), + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG), createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG), /* Unsigned Long configs */ - createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG), - createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG), + createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG), + createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG), /* Long Long configs */ - createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG), - createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG), - createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG), - createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG), - createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG), + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG), + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG), /* Size_t configs */ - createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG), - createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), - createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG), - createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), - createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG), + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG), + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG), /* NULL Terminator */ {NULL} From 82e997c043637ffbfe4fb233bf36abc798b3d863 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 26 Nov 2019 16:52:28 +0200 Subject: [PATCH 303/320] Additional config.c refactory and bugfixes - add capability for each config to have a callback to check if value is valid and return error string will enable converting many of the remaining custom configs into generic ones (reducing the x4 repetition for set,get,config,rewrite) - add capability for each config to to run some update code after config is changed (only for CONFIG SET) will also enable converting many of the remaining custom configs into generic ones - add capability to move default values from server.h and server.c to config.c will reduce many excess lines in server.h and server.c (plus, no need to rebuild the entire code base when a default change 8-)) other behavior changes: - fix bug in bool config get (always returning 'yes') - fix a bug in modifying jemalloc-bg-thread at runtime (didn't call set_jemalloc_bg_thread, due to bad merge conflict resolution (my fault)) - side effect when a failed attempt to enable activedefrag at runtime, we now respond with -ERR and not with -DISABLED --- src/config.c | 365 +++++++++++++++++++++++++++++++-------------------- src/server.c | 3 +- src/server.h | 2 +- 3 files changed, 226 insertions(+), 144 deletions(-) diff --git a/src/config.c b/src/config.c index 77dd94d62..516e62984 100644 --- a/src/config.c +++ b/src/config.c @@ -110,11 +110,15 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { typedef struct boolConfigData { int *config; /* The pointer to the server config this value is stored in */ const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int,char**); /* Optional function to check validity of new value */ + void (*update_fn)(int); /* Optional function to apply new value at runtime (only for CONFIG SET) */ } boolConfigData; typedef struct stringConfigData { char **config; /* Pointer to the server config this value is stored in. */ const char *default_value; /* Default value of the config on rewrite. */ + int (*is_valid_fn)(char*,char**); /* Optional function to check validity of new value */ + void (*update_fn)(char*); /* Optional function to apply new value at runtime (only for CONFIG SET) */ int convert_empty_to_null; /* Boolean indicating if empty strings should be stored as a NULL value. */ } stringConfigData; @@ -123,6 +127,8 @@ typedef struct enumConfigData { int *config; /* The pointer to the server config this value is stored in */ configEnum *enum_value; /* The underlying enum type this data represents */ const int default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(int,char**); /* Optional function to check validity of new value */ + void (*update_fn)(int); /* Optional function to apply new value at runtime (only for CONFIG SET) */ } enumConfigData; typedef enum numericType { @@ -144,6 +150,8 @@ typedef struct numericConfigData { long long lower_bound; /* The lower bound of this numeric value */ long long upper_bound; /* The upper bound of this numeric value */ const long long default_value; /* The default value of the config on rewrite */ + int (*is_valid_fn)(long long,char**); /* Optional function to check validity of new value */ + void (*update_fn)(long long); /* Optional function to apply new value at runtime (only for CONFIG SET) */ } numericConfigData; typedef union typeData { @@ -154,10 +162,12 @@ typedef union typeData { } typeData; typedef struct typeInterface { + /* Called on server start, to init the server with default value */ + void (*init)(typeData data); /* Called on server start, should return 1 on success, 0 on error and should set err */ int (*load)(typeData data, sds *argc, int argv, char **err); /* Called on CONFIG SET, returns 1 on success, 0 on error */ - int (*set)(typeData data, sds value); + int (*set)(typeData data, sds value, char **err); /* Called on CONFIG GET, required to add output to the client */ void (*get)(client *c, typeData data); /* Called on CONFIG REWRITE, required to rewrite the config state */ @@ -245,6 +255,12 @@ void queueLoadModule(sds path, sds *argv, int argc) { listAddNodeTail(server.loadmodule_queue,loadmod); } +void initConfigValues() { + for (standardConfig *config = configs; config->name != NULL; config++) { + config->interface.init(config->data); + } +} + void loadServerConfigFromString(char *config) { char *err = NULL; int linenum = 0, totlines, i; @@ -383,15 +399,6 @@ void loadServerConfigFromString(char *config) { err = "repl-backlog-ttl can't be negative "; goto loaderr; } - } else if (!strcasecmp(argv[0],"activedefrag") && argc == 2) { - if ((server.active_defrag_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - if (server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - err = "active defrag can't be enabled without proper jemalloc support"; goto loaderr; -#endif - } } else if (!strcasecmp(argv[0],"hz") && argc == 2) { server.config_hz = atoi(argv[1]); if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; @@ -671,6 +678,7 @@ void configSetCommand(client *c) { robj *o; long long ll; int err; + char *errstr = NULL; serverAssertWithInfo(c,c->argv[2],sdsEncodedObject(c->argv[2])); serverAssertWithInfo(c,c->argv[3],sdsEncodedObject(c->argv[3])); o = c->argv[3]; @@ -680,7 +688,7 @@ void configSetCommand(client *c) { if(config->modifiable && (!strcasecmp(c->argv[2]->ptr,config->name) || (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { - if (!config->interface.set(config->data,o->ptr)) { + if (!config->interface.set(config->data,o->ptr, &errstr)) { goto badfmt; } addReply(c,shared.ok); @@ -836,21 +844,6 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - /* Boolean fields. - * config_set_bool_field(name,var). */ - } config_set_bool_field( - "activedefrag",server.active_defrag_enabled) { -#ifndef HAVE_DEFRAG - if (server.active_defrag_enabled) { - server.active_defrag_enabled = 0; - addReplyError(c, - "-DISABLED Active defragmentation cannot be enabled: it " - "requires a Redis server compiled with a modified Jemalloc " - "like the one shipped by default with the Redis source " - "distribution"); - return; - } -#endif /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ } config_set_numerical_field( @@ -1002,9 +995,16 @@ void configSetCommand(client *c) { return; badfmt: /* Bad format errors */ - addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", - (char*)o->ptr, - (char*)c->argv[2]->ptr); + if (errstr) { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s' - %s", + (char*)o->ptr, + (char*)c->argv[2]->ptr, + errstr); + } else { + addReplyErrorFormat(c,"Invalid argument '%s' for CONFIG SET '%s'", + (char*)o->ptr, + (char*)c->argv[2]->ptr); + } } /*----------------------------------------------------------------------------- @@ -1092,7 +1092,6 @@ void configGetCommand(client *c) { config_get_numerical_field("hz",server.config_hz); /* Bool (yes/no) values */ - config_get_bool_field("activedefrag", server.active_defrag_enabled); config_get_bool_field("tls-cluster",server.tls_cluster); config_get_bool_field("tls-replication",server.tls_replication); config_get_bool_field("tls-auth-clients",server.tls_auth_clients); @@ -1815,7 +1814,6 @@ int rewriteConfig(char *path) { rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); - rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); @@ -1866,7 +1864,8 @@ static char loadbuf[LOADBUF_SIZE]; .alias = (config_alias), \ .modifiable = (is_modifiable), -#define embedConfigInterface(loadfn, setfn, getfn, rewritefn) .interface = { \ +#define embedConfigInterface(initfn, loadfn, setfn, getfn, rewritefn) .interface = { \ + .init = (initfn), \ .load = (loadfn), \ .set = (setfn), \ .get = (getfn), \ @@ -1886,48 +1885,72 @@ static char loadbuf[LOADBUF_SIZE]; */ /* Bool Configs */ +static void boolConfigInit(typeData data) { + *data.yesno.config = data.yesno.default_value; +} + static int boolConfigLoad(typeData data, sds *argv, int argc, char **err) { + int yn; if (argc != 2) { *err = "wrong number of arguments"; return 0; } - if ((*(data.yesno.config) = yesnotoi(argv[1])) == -1) { + if ((yn = yesnotoi(argv[1])) == -1) { *err = "argument must be 'yes' or 'no'"; return 0; } + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; + *data.yesno.config = yn; return 1; } -static int boolConfigSet(typeData data, sds value) { +static int boolConfigSet(typeData data, sds value, char **err) { int yn = yesnotoi(value); if (yn == -1) return 0; + if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) + return 0; *(data.yesno.config) = yn; + if (data.yesno.update_fn) + data.yesno.update_fn(yn); return 1; } static void boolConfigGet(client *c, typeData data) { - addReplyBulkCString(c, data.yesno.config ? "yes" : "no"); + addReplyBulkCString(c, *data.yesno.config ? "yes" : "no"); } static void boolConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { rewriteConfigYesNoOption(state, name,*(data.yesno.config), data.yesno.default_value); } -#define createBoolConfig(name, alias, modifiable, config_addr, default) { \ +#define createBoolConfig(name, alias, modifiable, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ + embedConfigInterface(boolConfigInit, boolConfigLoad, boolConfigSet, boolConfigGet, boolConfigRewrite) \ .data.yesno = { \ .config = &(config_addr), \ - .default_value = (default) \ + .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ } \ } /* String Configs */ +static void stringConfigInit(typeData data) { + if (data.string.convert_empty_to_null) { + *data.string.config = data.string.default_value ? zstrdup(data.string.default_value) : NULL; + } else { + *data.string.config = zstrdup(data.string.default_value); + } +} + static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { if (argc != 2) { *err = "wrong number of arguments"; return 0; } + if (data.string.is_valid_fn && !data.string.is_valid_fn(argv[1], err)) + return 0; zfree(*data.string.config); if (data.string.convert_empty_to_null) { *data.string.config = argv[1][0] ? zstrdup(argv[1]) : NULL; @@ -1937,13 +1960,17 @@ static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { return 1; } -static int stringConfigSet(typeData data, sds value) { +static int stringConfigSet(typeData data, sds value, char **err) { + if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) + return 0; zfree(*data.string.config); if (data.string.convert_empty_to_null) { *data.string.config = value[0] ? zstrdup(value) : NULL; } else { *data.string.config = zstrdup(value); } + if (data.string.update_fn) + data.string.update_fn(*data.string.config); return 1; } @@ -1958,17 +1985,23 @@ static void stringConfigRewrite(typeData data, const char *name, struct rewriteC #define ALLOW_EMPTY_STRING 0 #define EMPTY_STRING_IS_NULL 1 -#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default) { \ +#define createStringConfig(name, alias, modifiable, empty_to_null, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ + embedConfigInterface(stringConfigInit, stringConfigLoad, stringConfigSet, stringConfigGet, stringConfigRewrite) \ .data.string = { \ .config = &(config_addr), \ .default_value = (default), \ - .convert_empty_to_null = (empty_to_null) \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .convert_empty_to_null = (empty_to_null), \ } \ } /* Enum configs */ +static void configEnumInit(typeData data) { + *data.enumd.config = data.enumd.default_value; +} + static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { if (argc != 2) { *err = "wrong number of arguments"; @@ -1995,14 +2028,20 @@ static int configEnumLoad(typeData data, sds *argv, int argc, char **err) { *err = loadbuf; return 0; } + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; *(data.enumd.config) = enumval; return 1; } -static int configEnumSet(typeData data, sds value) { +static int configEnumSet(typeData data, sds value, char **err) { int enumval = configEnumGetValue(data.enumd.enum_value, value); if (enumval == INT_MIN) return 0; + if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) + return 0; *(data.enumd.config) = enumval; + if (data.enumd.update_fn) + data.enumd.update_fn(enumval); return 1; } @@ -2014,17 +2053,31 @@ static void configEnumRewrite(typeData data, const char *name, struct rewriteCon rewriteConfigEnumOption(state, name,*(data.enumd.config), data.enumd.enum_value, data.enumd.default_value); } -#define createEnumConfig(name, alias, modifiable, enum, config_addr, default) { \ +#define createEnumConfig(name, alias, modifiable, enum, config_addr, default, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ + embedConfigInterface(configEnumInit, configEnumLoad, configEnumSet, configEnumGet, configEnumRewrite) \ .data.enumd = { \ .config = &(config_addr), \ .default_value = (default), \ - .enum_value = (enum) \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ + .enum_value = (enum), \ } \ } /* Numeric configs */ +static void numericConfigInit(typeData data) { + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { + *(data.numeric.config.i) = (int) data.numeric.default_value; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { + *(data.numeric.config.ul) = (unsigned long) data.numeric.default_value; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { + *(data.numeric.config.ll) = (long long) data.numeric.default_value; + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + *(data.numeric.config.st) = (size_t) data.numeric.default_value; + } +} + static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { long long ll; @@ -2057,6 +2110,9 @@ static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { return 0; } + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { *(data.numeric.config.i) = (int) ll; } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { @@ -2070,7 +2126,7 @@ static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { return 1; } -static int numericConfigSet(typeData data, sds value) { +static int numericConfigSet(typeData data, sds value, char **err) { long long ll; if (data.numeric.is_memory) { int memerr; @@ -2085,6 +2141,9 @@ static int numericConfigSet(typeData data, sds value) { return 0; } + if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) + return 0; + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { *(data.numeric.config.i) = (int) ll; } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { @@ -2095,6 +2154,8 @@ static int numericConfigSet(typeData data, sds value) { *(data.numeric.config.st) = (size_t) ll; } + if (data.numeric.update_fn) + data.numeric.update_fn(ll); return 1; } @@ -2138,142 +2199,162 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite #define INTEGER_CONFIG 0 #define MEMORY_CONFIG 1 -#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) { \ +#define embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) { \ embedCommonConfig(name, alias, modifiable) \ - embedConfigInterface(numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ + embedConfigInterface(numericConfigInit, numericConfigLoad, numericConfigSet, numericConfigGet, numericConfigRewrite) \ .data.numeric = { \ .lower_bound = (lower), \ .upper_bound = (upper), \ .default_value = (default), \ + .is_valid_fn = (is_valid), \ + .update_fn = (update), \ .is_memory = (memory), -#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ +#define createIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_INT, \ .config.i = &(config_addr) \ } \ } -#define createUnsignedLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ +#define createUnsignedLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_UNSIGNED_LONG, \ .config.ul = &(config_addr) \ } \ } -#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ +#define createLongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_LONG_LONG, \ .config.ll = &(config_addr) \ } \ } -#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ - embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory) \ +#define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_SIZE_T, \ .config.st = &(config_addr) \ } \ } +int is_active_defrag_valid(int val, char **err) { +#ifndef HAVE_DEFRAG + if (val) { + *err = "Active defragmentation cannot be enabled: it " + "requires a Redis server compiled with a modified Jemalloc " + "like the one shipped by default with the Redis source " + "distribution"; + return 0; + } +#else + UNUSED(val); + UNUSED(err); +#endif + return 1; +} + + standardConfig configs[] = { /* Bool configs */ - createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM), - createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0), - createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, CONFIG_DEFAULT_IO_THREADS_DO_READS), - createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1), - createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO), - createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE), - createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION), - createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING), - createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR), - createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ), - createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION), - createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE), - createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL), - createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY), - createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC), - createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED), - createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC), - createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE), - createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE), - createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC), - createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED), - createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE), - createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER), - createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH), - createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA), - createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY), - createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY), - createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1), + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM ,NULL, NULL), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0 ,NULL, NULL), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, CONFIG_DEFAULT_IO_THREADS_DO_READS ,NULL, NULL), + createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1 ,NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO ,NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE ,NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION ,NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING ,NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR ,NULL, NULL), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ ,NULL, NULL), + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION ,NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE ,NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL ,NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY ,NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC ,NULL, NULL), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED ,NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC ,NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE ,NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE ,NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC ,NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED ,NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE ,NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER ,NULL, NULL), + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH ,NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA ,NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY ,NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY ,NULL, NULL), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, set_jemalloc_bg_thread), + createBoolConfig("activedefrag",NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, is_active_defrag_valid, NULL), /* String Configs */ - createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME), - createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL), - createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE), - createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL), - createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL), + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME ,NULL, NULL), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL ,NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE ,NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP ,NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL ,NULL, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL ,NULL, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL ,NULL, NULL), /* Enum Configs */ - createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE), - createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0), - createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD), - createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY), - createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY), - createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC), + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE ,NULL, NULL), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0 ,NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD ,NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY ,NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY ,NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC ,NULL, NULL), /* Integer configs */ - createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG), - createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG), - createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG), - createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG), - createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG), - createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG), - createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG), - createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG), - createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG), - createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG), - createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG), - createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG), - createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG), - createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG), - createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG), - createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG), - createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG), - createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG), - createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG), - createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG), - createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG), - createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG), - createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG), - createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG), - createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG), + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG ,NULL, NULL), /* Unsigned Long configs */ - createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG), - createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG), + createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG ,NULL, NULL), + createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG ,NULL, NULL), /* Long Long configs */ - createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG), - createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG), - createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG), - createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG), - createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG), + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG ,NULL, NULL), /* Size_t configs */ - createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG), - createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG), - createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), - createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG), - createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG), - createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG), + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG ,NULL, NULL), /* NULL Terminator */ {NULL} diff --git a/src/server.c b/src/server.c index 803cfc809..497d093f1 100644 --- a/src/server.c +++ b/src/server.c @@ -2295,7 +2295,6 @@ void initServerConfig(void) { server.active_expire_enabled = 1; server.active_expire_effort = CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT; server.jemalloc_bg_thread = 1; - server.active_defrag_enabled = CONFIG_DEFAULT_ACTIVE_DEFRAG; server.active_defrag_ignore_bytes = CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES; server.active_defrag_threshold_lower = CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER; server.active_defrag_threshold_upper = CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER; @@ -2485,6 +2484,8 @@ void initServerConfig(void) { * script to the slave / AOF. This is the new way starting from * Redis 5. However it is possible to revert it via redis.conf. */ server.lua_always_replicate_commands = 1; + + initConfigValues(); } extern char **environ; diff --git a/src/server.h b/src/server.h index 547e7a4bf..f31cb8fc1 100644 --- a/src/server.h +++ b/src/server.h @@ -171,7 +171,6 @@ typedef long long ustime_t; /* microsecond time type. */ #define CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE 0 #define CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL 0 #define CONFIG_DEFAULT_ALWAYS_SHOW_LOGO 0 -#define CONFIG_DEFAULT_ACTIVE_DEFRAG 0 #define CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER 10 /* don't defrag when fragmentation is below 10% */ #define CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER 100 /* maximum defrag force at 100% fragmentation */ #define CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES (100<<20) /* don't defrag if frag overhead is below 100mb */ @@ -2077,6 +2076,7 @@ void resetServerSaveParams(void); struct rewriteConfigState; /* Forward declaration to export API. */ void rewriteConfigRewriteLine(struct rewriteConfigState *state, const char *option, sds line, int force); int rewriteConfig(char *path); +void initConfigValues(); /* db.c -- Keyspace access API */ int removeExpire(redisDb *db, robj *key); From 0462896e15b67d82283c8d31c4ec482811b01f74 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 28 Nov 2019 11:11:07 +0200 Subject: [PATCH 304/320] More improvements and fixes to generic config infra - Adding is_valid_fn and update_fn, both return 1 for success and 0 for failure with an optional error message. - Bugfix in handling boundary check of unsigned numeric types (was boundaries as signed) - Adding more numeric types to generic mechanism: uint, ulonglong, long, time_t, off_t - More verbose error replies ("argument must be between" in out of range CONFIG SET (like config file parsing) --- src/config.c | 305 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 221 insertions(+), 84 deletions(-) diff --git a/src/config.c b/src/config.c index 516e62984..e27a17d7d 100644 --- a/src/config.c +++ b/src/config.c @@ -105,20 +105,32 @@ clientBufferLimitsConfig clientBufferLimitsDefaults[CLIENT_TYPE_OBUF_COUNT] = { {1024*1024*32, 1024*1024*8, 60} /* pubsub */ }; +/* Generic config infrastructure function pointers + * int is_valid_fn(val, err) + * Return 1 when val is valid, and 0 when invalid. + * Optionslly set err to a static error string. + * int update_fn(val, prev, err) + * This function is called only for CONFIG SET command (not at config file parsing) + * It is called after the actual config is applied, + * Return 1 for success, and 0 for failure. + * Optionslly set err to a static error string. + * On failure the config change will be reverted. + */ + /* Configuration values that require no special handling to set, get, load or * rewrite. */ typedef struct boolConfigData { int *config; /* The pointer to the server config this value is stored in */ const int default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(int,char**); /* Optional function to check validity of new value */ - void (*update_fn)(int); /* Optional function to apply new value at runtime (only for CONFIG SET) */ + int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ } boolConfigData; typedef struct stringConfigData { char **config; /* Pointer to the server config this value is stored in. */ const char *default_value; /* Default value of the config on rewrite. */ - int (*is_valid_fn)(char*,char**); /* Optional function to check validity of new value */ - void (*update_fn)(char*); /* Optional function to apply new value at runtime (only for CONFIG SET) */ + int (*is_valid_fn)(char* val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(char* val, char* prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ int convert_empty_to_null; /* Boolean indicating if empty strings should be stored as a NULL value. */ } stringConfigData; @@ -127,31 +139,43 @@ typedef struct enumConfigData { int *config; /* The pointer to the server config this value is stored in */ configEnum *enum_value; /* The underlying enum type this data represents */ const int default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(int,char**); /* Optional function to check validity of new value */ - void (*update_fn)(int); /* Optional function to apply new value at runtime (only for CONFIG SET) */ + int (*is_valid_fn)(int val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(int val, int prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ } enumConfigData; typedef enum numericType { NUMERIC_TYPE_INT, + NUMERIC_TYPE_UINT, + NUMERIC_TYPE_LONG, + NUMERIC_TYPE_ULONG, NUMERIC_TYPE_LONG_LONG, - NUMERIC_TYPE_UNSIGNED_LONG, - NUMERIC_TYPE_SIZE_T + NUMERIC_TYPE_ULONG_LONG, + NUMERIC_TYPE_SIZE_T, + NUMERIC_TYPE_SSIZE_T, + NUMERIC_TYPE_OFF_T, + NUMERIC_TYPE_TIME_T, } numericType; typedef struct numericConfigData { union { int *i; - long long *ll; + unsigned int *ui; + long *l; unsigned long *ul; + long long *ll; + unsigned long long *ull; size_t *st; + ssize_t *sst; + off_t *ot; + time_t *tt; } config; /* The pointer to the numeric config this value is stored in */ int is_memory; /* Indicates if this value can be loaded as a memory value */ numericType numeric_type; /* An enum indicating the type of this value */ long long lower_bound; /* The lower bound of this numeric value */ long long upper_bound; /* The upper bound of this numeric value */ const long long default_value; /* The default value of the config on rewrite */ - int (*is_valid_fn)(long long,char**); /* Optional function to check validity of new value */ - void (*update_fn)(long long); /* Optional function to apply new value at runtime (only for CONFIG SET) */ + int (*is_valid_fn)(long long val, char **err); /* Optional function to check validity of new value (generic doc above) */ + int (*update_fn)(long long val, long long prev, char **err); /* Optional function to apply new value at runtime (generic doc above) */ } numericConfigData; typedef union typeData { @@ -1910,9 +1934,12 @@ static int boolConfigSet(typeData data, sds value, char **err) { if (yn == -1) return 0; if (data.yesno.is_valid_fn && !data.yesno.is_valid_fn(yn, err)) return 0; + int prev = *(data.yesno.config); *(data.yesno.config) = yn; - if (data.yesno.update_fn) - data.yesno.update_fn(yn); + if (data.yesno.update_fn && !data.yesno.update_fn(yn, prev, err)) { + *(data.yesno.config) = prev; + return 0; + } return 1; } @@ -1963,14 +1990,18 @@ static int stringConfigLoad(typeData data, sds *argv, int argc, char **err) { static int stringConfigSet(typeData data, sds value, char **err) { if (data.string.is_valid_fn && !data.string.is_valid_fn(value, err)) return 0; - zfree(*data.string.config); + char *prev = *data.string.config; if (data.string.convert_empty_to_null) { *data.string.config = value[0] ? zstrdup(value) : NULL; } else { *data.string.config = zstrdup(value); } - if (data.string.update_fn) - data.string.update_fn(*data.string.config); + if (data.string.update_fn && !data.string.update_fn(*data.string.config, prev, err)) { + zfree(*data.string.config); + *data.string.config = prev; + return 0; + } + zfree(prev); return 1; } @@ -2039,9 +2070,12 @@ static int configEnumSet(typeData data, sds value, char **err) { if (enumval == INT_MIN) return 0; if (data.enumd.is_valid_fn && !data.enumd.is_valid_fn(enumval, err)) return 0; + int prev = *(data.enumd.config); *(data.enumd.config) = enumval; - if (data.enumd.update_fn) - data.enumd.update_fn(enumval); + if (data.enumd.update_fn && !data.enumd.update_fn(enumval, prev, err)) { + *(data.enumd.config) = prev; + return 0; + } return 1; } @@ -2065,17 +2099,59 @@ static void configEnumRewrite(typeData data, const char *name, struct rewriteCon } \ } +/* Gets a 'long long val' and sets it into the union, using a macro to get + * compile time type check. */ +#define SET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + *(data.numeric.config.i) = (int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + *(data.numeric.config.ui) = (unsigned int) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + *(data.numeric.config.l) = (long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + *(data.numeric.config.ul) = (unsigned long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + *(data.numeric.config.ll) = (long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + *(data.numeric.config.ull) = (unsigned long long) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + *(data.numeric.config.st) = (size_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + *(data.numeric.config.sst) = (ssize_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + *(data.numeric.config.ot) = (off_t) val; \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + *(data.numeric.config.tt) = (time_t) val; \ + } + +/* Gets a 'long long val' and sets it with the value from the union, using a + * macro to get compile time type check. */ +#define GET_NUMERIC_TYPE(val) \ + if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { \ + val = *(data.numeric.config.i); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_UINT) { \ + val = *(data.numeric.config.ui); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG) { \ + val = *(data.numeric.config.l); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG) { \ + val = *(data.numeric.config.ul); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { \ + val = *(data.numeric.config.ll); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG) { \ + val = *(data.numeric.config.ull); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { \ + val = *(data.numeric.config.st); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_SSIZE_T) { \ + val = *(data.numeric.config.sst); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_OFF_T) { \ + val = *(data.numeric.config.ot); \ + } else if (data.numeric.numeric_type == NUMERIC_TYPE_TIME_T) { \ + val = *(data.numeric.config.tt); \ + } + /* Numeric configs */ static void numericConfigInit(typeData data) { - if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { - *(data.numeric.config.i) = (int) data.numeric.default_value; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { - *(data.numeric.config.ul) = (unsigned long) data.numeric.default_value; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { - *(data.numeric.config.ll) = (long long) data.numeric.default_value; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - *(data.numeric.config.st) = (size_t) data.numeric.default_value; - } + SET_NUMERIC_TYPE(data.numeric.default_value) } static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { @@ -2100,34 +2176,44 @@ static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { } } - if (ll > data.numeric.upper_bound || - ll < data.numeric.lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %lld and %lld inclusive", - data.numeric.lower_bound, - data.numeric.upper_bound); - *err = loadbuf; - return 0; + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || + ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } } if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) return 0; - if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { - *(data.numeric.config.i) = (int) ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { - *(data.numeric.config.ul) = (unsigned long) ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { - *(data.numeric.config.ll) = ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - *(data.numeric.config.st) = (size_t) ll; - } + SET_NUMERIC_TYPE(ll) return 1; } static int numericConfigSet(typeData data, sds value, char **err) { - long long ll; + long long ll, prev = 0; if (data.numeric.is_memory) { int memerr; ll = memtoll(value, &memerr); @@ -2136,26 +2222,44 @@ static int numericConfigSet(typeData data, sds value, char **err) { if (!string2ll(value, sdslen(value),&ll)) return 0; } - if (ll > data.numeric.upper_bound || - ll < data.numeric.lower_bound) { - return 0; + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || + ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } } if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) return 0; - if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { - *(data.numeric.config.i) = (int) ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { - *(data.numeric.config.ul) = (unsigned long) ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { - *(data.numeric.config.ll) = ll; - } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - *(data.numeric.config.st) = (size_t) ll; - } + GET_NUMERIC_TYPE(prev) + SET_NUMERIC_TYPE(ll) - if (data.numeric.update_fn) - data.numeric.update_fn(ll); + if (data.numeric.update_fn && !data.numeric.update_fn(ll, prev, err)) { + SET_NUMERIC_TYPE(prev) + return 0; + } return 1; } @@ -2163,15 +2267,7 @@ static void numericConfigGet(client *c, typeData data) { char buf[128]; long long value = 0; - if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { - value = *(data.numeric.config.i); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { - value = *(data.numeric.config.ul); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { - value = *(data.numeric.config.ll); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - value = *(data.numeric.config.st); - } + GET_NUMERIC_TYPE(value) ll2string(buf, sizeof(buf), value); addReplyBulkCString(c, buf); @@ -2179,15 +2275,8 @@ static void numericConfigGet(client *c, typeData data) { static void numericConfigRewrite(typeData data, const char *name, struct rewriteConfigState *state) { long long value = 0; - if (data.numeric.numeric_type == NUMERIC_TYPE_INT) { - value = *(data.numeric.config.i); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_UNSIGNED_LONG) { - value = *(data.numeric.config.ul); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_LONG_LONG) { - value = *(data.numeric.config.ll); - } else if (data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - value = *(data.numeric.config.st); - } + + GET_NUMERIC_TYPE(value) if (data.numeric.is_memory) { rewriteConfigBytesOption(state, name, value, data.numeric.default_value); @@ -2217,9 +2306,23 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite } \ } -#define createUnsignedLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ +#define createUIntConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ - .numeric_type = NUMERIC_TYPE_UNSIGNED_LONG, \ + .numeric_type = NUMERIC_TYPE_UINT, \ + .config.ui = &(config_addr) \ + } \ +} + +#define createLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_LONG, \ + .config.l = &(config_addr) \ + } \ +} + +#define createULongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG, \ .config.ul = &(config_addr) \ } \ } @@ -2231,6 +2334,13 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite } \ } +#define createULongLongConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_ULONG_LONG, \ + .config.ull = &(config_addr) \ + } \ +} + #define createSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ .numeric_type = NUMERIC_TYPE_SIZE_T, \ @@ -2238,7 +2348,28 @@ static void numericConfigRewrite(typeData data, const char *name, struct rewrite } \ } -int is_active_defrag_valid(int val, char **err) { +#define createSSizeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_SSIZE_T, \ + .config.sst = &(config_addr) \ + } \ +} + +#define createTimeTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_TIME_T, \ + .config.tt = &(config_addr) \ + } \ +} + +#define createOffTConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + embedCommonNumericalConfig(name, alias, modifiable, lower, upper, config_addr, default, memory, is_valid, update) \ + .numeric_type = NUMERIC_TYPE_OFF_T, \ + .config.ot = &(config_addr) \ + } \ +} + +static int isValidActiveDefrag(int val, char **err) { #ifndef HAVE_DEFRAG if (val) { *err = "Active defragmentation cannot be enabled: it " @@ -2254,6 +2385,12 @@ int is_active_defrag_valid(int val, char **err) { return 1; } +static int updateJemallocBgThread(int val, int prev, char **err) { + UNUSED(prev); + UNUSED(err); + set_jemalloc_bg_thread(val); + return 1; +} standardConfig configs[] = { /* Bool configs */ @@ -2284,8 +2421,8 @@ standardConfig configs[] = { createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA ,NULL, NULL), createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY ,NULL, NULL), createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY ,NULL, NULL), - createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, set_jemalloc_bg_thread), - createBoolConfig("activedefrag",NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, is_active_defrag_valid, NULL), + createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), + createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), /* String Configs */ createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME ,NULL, NULL), @@ -2336,8 +2473,8 @@ standardConfig configs[] = { createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG ,NULL, NULL), /* Unsigned Long configs */ - createUnsignedLongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG ,NULL, NULL), - createUnsignedLongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG ,NULL, NULL), + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG ,NULL, NULL), + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG ,NULL, NULL), /* Long Long configs */ createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG ,NULL, NULL), From 493cc493b69cba6c69e96603705899e511f9df76 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 28 Nov 2019 11:24:57 +0200 Subject: [PATCH 305/320] Converting more configs to use generic infra, and moving defaults to config.c Changes in behavior: - Change server.stream_node_max_entries from int64_t to long long, so that it can be used by the generic infra - standard error reply instead of "repl-backlog-size must be 1 or greater" and such - tls-port and a few TLS booleans were readable (config get) even when USE_OPENSSL was off (now they aren't) - syslog-enabled, syslog-ident, cluster-enabled, appendfilename, and supervised didn't have a get (now they do) - pidfile was initialized to NULL in InitServerConfig but had CONFIG_DEFAULT_PID_FILE in rewriteConfig (so the real default was "", but rewrite would cause it to be set), fixed the rewrite. - TLS config in server.h was uninitialized (if no tls config args were provided) Adding test for sanity and coverage --- src/cluster.h | 5 - src/config.c | 655 ++++++++++++----------------------- src/server.c | 105 +----- src/server.h | 96 +---- tests/unit/introspection.tcl | 65 ++++ 5 files changed, 294 insertions(+), 632 deletions(-) diff --git a/src/cluster.h b/src/cluster.h index ffbb29f0d..3ba60df6e 100644 --- a/src/cluster.h +++ b/src/cluster.h @@ -13,15 +13,10 @@ /* The following defines are amount of time, sometimes expressed as * multiplicators of the node timeout value (when ending with MULT). */ -#define CLUSTER_DEFAULT_NODE_TIMEOUT 15000 -#define CLUSTER_DEFAULT_SLAVE_VALIDITY 10 /* Slave max data age factor. */ -#define CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE 1 -#define CLUSTER_DEFAULT_SLAVE_NO_FAILOVER 0 /* Failover by default. */ #define CLUSTER_FAIL_REPORT_VALIDITY_MULT 2 /* Fail report validity. */ #define CLUSTER_FAIL_UNDO_TIME_MULT 2 /* Undo fail if master is back. */ #define CLUSTER_FAIL_UNDO_TIME_ADD 10 /* Some additional time. */ #define CLUSTER_FAILOVER_DELAY 5 /* Seconds */ -#define CLUSTER_DEFAULT_MIGRATION_BARRIER 1 #define CLUSTER_MF_TIMEOUT 5000 /* Milliseconds to do a manual failover. */ #define CLUSTER_MF_PAUSE_MULT 2 /* Master pause manual failover mult. */ #define CLUSTER_SLAVE_MIGRATION_DELAY 5000 /* Delay for slave migration. */ diff --git a/src/config.c b/src/config.c index e27a17d7d..16cc74f57 100644 --- a/src/config.c +++ b/src/config.c @@ -386,22 +386,8 @@ void loadServerConfigFromString(char *config) { } fclose(logfp); } - } else if (!strcasecmp(argv[0],"syslog-enabled") && argc == 2) { - if ((server.syslog_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"syslog-ident") && argc == 2) { - if (server.syslog_ident) zfree(server.syslog_ident); - server.syslog_ident = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"include") && argc == 2) { loadServerConfig(argv[1],NULL); - } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { - server.maxclients = atoi(argv[1]); - if (server.maxclients < 1) { - err = "Invalid max clients limit"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { - server.maxmemory = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"client-query-buffer-limit")) && argc == 2) { server.client_max_querybuf_len = memtoll(argv[1],NULL); } else if ((!strcasecmp(argv[0],"slaveof") || @@ -410,39 +396,6 @@ void loadServerConfigFromString(char *config) { server.masterhost = sdsnew(argv[1]); server.masterport = atoi(argv[2]); server.repl_state = REPL_STATE_CONNECT; - } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = memtoll(argv[1],NULL); - if (size <= 0) { - err = "repl-backlog-size must be 1 or greater."; - goto loaderr; - } - resizeReplicationBacklog(size); - } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { - server.repl_backlog_time_limit = atoi(argv[1]); - if (server.repl_backlog_time_limit < 0) { - err = "repl-backlog-ttl can't be negative "; - goto loaderr; - } - } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - server.config_hz = atoi(argv[1]); - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; - } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { - if ((server.aof_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - server.aof_state = server.aof_enabled ? AOF_ON : AOF_OFF; - } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "appendfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.aof_filename); - server.aof_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"auto-aof-rewrite-min-size") && - argc == 2) - { - server.aof_rewrite_min_size = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > CONFIG_AUTHPASS_MAX_LEN) { err = "Password is longer than CONFIG_AUTHPASS_MAX_LEN"; @@ -454,15 +407,6 @@ void loadServerConfigFromString(char *config) { sds aclop = sdscatprintf(sdsempty(),">%s",argv[1]); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { - if (!pathIsBaseName(argv[1])) { - err = "dbfilename can't be a path, just a filename"; - goto loaderr; - } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"stream-node-max-entries") && argc == 2) { - server.stream_node_max_entries = atoi(argv[1]); } else if (!strcasecmp(argv[0],"list-max-ziplist-entries") && argc == 2){ /* DEAD OPTION */ } else if (!strcasecmp(argv[0],"list-max-ziplist-value") && argc == 2) { @@ -491,10 +435,6 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((server.cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { zfree(server.cluster_configfile); server.cluster_configfile = zstrdup(argv[1]); @@ -520,20 +460,6 @@ void loadServerConfigFromString(char *config) { server.client_obuf_limits[class].hard_limit_bytes = hard; server.client_obuf_limits[class].soft_limit_bytes = soft; server.client_obuf_limits[class].soft_limit_seconds = soft_seconds; - } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || - !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) - { - server.repl_min_slaves_to_write = atoi(argv[1]); - if (server.repl_min_slaves_to_write < 0) { - err = "Invalid value for min-replicas-to-write."; goto loaderr; - } - } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || - !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) - { - server.repl_min_slaves_max_lag = atoi(argv[1]); - if (server.repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-replicas-max-lag."; goto loaderr; - } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -565,45 +491,6 @@ void loadServerConfigFromString(char *config) { err = sentinelHandleConfiguration(argv+1,argc-1); if (err) goto loaderr; } -#ifdef USE_OPENSSL - } else if (!strcasecmp(argv[0],"tls-port") && argc == 2) { - server.tls_port = atoi(argv[1]); - if (server.port < 0 || server.port > 65535) { - err = "Invalid tls-port"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"tls-cluster") && argc == 2) { - server.tls_cluster = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"tls-replication") && argc == 2) { - server.tls_replication = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"tls-auth-clients") && argc == 2) { - server.tls_auth_clients = yesnotoi(argv[1]); - } else if (!strcasecmp(argv[0],"tls-cert-file") && argc == 2) { - zfree(server.tls_ctx_config.cert_file); - server.tls_ctx_config.cert_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-key-file") && argc == 2) { - zfree(server.tls_ctx_config.key_file); - server.tls_ctx_config.key_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-dh-params-file") && argc == 2) { - zfree(server.tls_ctx_config.dh_params_file); - server.tls_ctx_config.dh_params_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-ca-cert-file") && argc == 2) { - zfree(server.tls_ctx_config.ca_cert_file); - server.tls_ctx_config.ca_cert_file = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-ca-cert-dir") && argc == 2) { - zfree(server.tls_ctx_config.ca_cert_dir); - server.tls_ctx_config.ca_cert_dir = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-protocols") && argc >= 2) { - zfree(server.tls_ctx_config.protocols); - server.tls_ctx_config.protocols = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-ciphers") && argc == 2) { - zfree(server.tls_ctx_config.ciphers); - server.tls_ctx_config.ciphers = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-ciphersuites") && argc == 2) { - zfree(server.tls_ctx_config.ciphersuites); - server.tls_ctx_config.ciphersuites = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"tls-prefer-server-ciphers") && argc == 2) { - server.tls_ctx_config.prefer_server_ciphers = yesnotoi(argv[1]); -#endif /* USE_OPENSSL */ } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } @@ -723,14 +610,7 @@ void configSetCommand(client *c) { if (0) { /* this starts the config_set macros else-if chain. */ /* Special fields that can't be handled with general macros. */ - config_set_special_field("dbfilename") { - if (!pathIsBaseName(o->ptr)) { - addReplyError(c, "dbfilename can't be a path, just a filename"); - return; - } - zfree(server.rdb_filename); - server.rdb_filename = zstrdup(o->ptr); - } config_set_special_field("requirepass") { + config_set_special_field("requirepass") { if (sdslen(o->ptr) > CONFIG_AUTHPASS_MAX_LEN) goto badfmt; /* The old "requirepass" directive just translates to setting * a password to the default user. */ @@ -738,46 +618,6 @@ void configSetCommand(client *c) { sds aclop = sdscatprintf(sdsempty(),">%s",(char*)o->ptr); ACLSetUser(DefaultUser,aclop,sdslen(aclop)); sdsfree(aclop); - } config_set_special_field("maxclients") { - int orig_value = server.maxclients; - - if (getLongLongFromObject(o,&ll) == C_ERR || ll < 1) goto badfmt; - - /* Try to check if the OS is capable of supporting so many FDs. */ - server.maxclients = ll; - if (ll > orig_value) { - adjustOpenFilesLimit(); - if (server.maxclients != ll) { - addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); - server.maxclients = orig_value; - return; - } - if ((unsigned int) aeGetSetSize(server.el) < - server.maxclients + CONFIG_FDSET_INCR) - { - if (aeResizeSetSize(server.el, - server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) - { - addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); - server.maxclients = orig_value; - return; - } - } - } - } config_set_special_field("appendonly") { - int enable = yesnotoi(o->ptr); - - if (enable == -1) goto badfmt; - server.aof_enabled = enable; - if (enable == 0 && server.aof_state != AOF_OFF) { - stopAppendOnly(); - } else if (enable && server.aof_state == AOF_OFF) { - if (startAppendOnly() == C_ERR) { - addReplyError(c, - "Unable to turn on AOF. Check server logs."); - return; - } - } } config_set_special_field("save") { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -870,28 +710,6 @@ void configSetCommand(client *c) { server.notify_keyspace_events = flags; /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ - } config_set_numerical_field( - "stream-node-max-entries",server.stream_node_max_entries,0,LLONG_MAX) { - } config_set_numerical_field( - "repl-backlog-ttl",server.repl_backlog_time_limit,0,LONG_MAX) { - } config_set_numerical_field( - "min-slaves-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-to-write",server.repl_min_slaves_to_write,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-slaves-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "min-replicas-max-lag",server.repl_min_slaves_max_lag,0,INT_MAX) { - refreshGoodSlavesCount(); - } config_set_numerical_field( - "hz",server.config_hz,0,INT_MAX) { - /* Hz is more an hint from the user, so we accept values out of range - * but cap them to reasonable values. */ - if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; - if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) @@ -900,113 +718,8 @@ void configSetCommand(client *c) { disableWatchdog(); /* Memory fields. * config_set_memory_field(name,var) */ - } config_set_memory_field("maxmemory",server.maxmemory) { - if (server.maxmemory) { - if (server.maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeededAndSafe(); - } } config_set_memory_field( "client-query-buffer-limit",server.client_max_querybuf_len) { - } config_set_memory_field("repl-backlog-size",ll) { - resizeReplicationBacklog(ll); - } config_set_memory_field("auto-aof-rewrite-min-size",ll) { - server.aof_rewrite_min_size = ll; -#ifdef USE_OPENSSL - /* TLS fields. */ - } config_set_special_field("tls-cert-file") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.cert_file = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-cert-file. Check server logs."); - return; - } - zfree(server.tls_ctx_config.cert_file); - server.tls_ctx_config.cert_file = zstrdup(o->ptr); - } config_set_special_field("tls-key-file") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.key_file = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-key-file. Check server logs."); - return; - } - zfree(server.tls_ctx_config.key_file); - server.tls_ctx_config.key_file = zstrdup(o->ptr); - } config_set_special_field("tls-dh-params-file") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.dh_params_file = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-dh-params-file. Check server logs."); - return; - } - zfree(server.tls_ctx_config.dh_params_file); - server.tls_ctx_config.dh_params_file = zstrdup(o->ptr); - } config_set_special_field("tls-ca-cert-file") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.ca_cert_file = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-ca-cert-file. Check server logs."); - return; - } - zfree(server.tls_ctx_config.ca_cert_file); - server.tls_ctx_config.ca_cert_file = zstrdup(o->ptr); - } config_set_special_field("tls-ca-cert-dir") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.ca_cert_dir = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-ca-cert-dir. Check server logs."); - return; - } - zfree(server.tls_ctx_config.ca_cert_dir); - server.tls_ctx_config.ca_cert_dir = zstrdup(o->ptr); - } config_set_bool_field("tls-auth-clients", server.tls_auth_clients) { - } config_set_bool_field("tls-replication", server.tls_replication) { - } config_set_bool_field("tls-cluster", server.tls_cluster) { - } config_set_special_field("tls-protocols") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.protocols = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-protocols. Check server logs."); - return; - } - zfree(server.tls_ctx_config.protocols); - server.tls_ctx_config.protocols = zstrdup(o->ptr); - } config_set_special_field("tls-ciphers") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.ciphers = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-ciphers. Check server logs."); - return; - } - zfree(server.tls_ctx_config.ciphers); - server.tls_ctx_config.ciphers = zstrdup(o->ptr); - } config_set_special_field("tls-ciphersuites") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.ciphersuites = (char *) o->ptr; - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, - "Unable to configure tls-ciphersuites. Check server logs."); - return; - } - zfree(server.tls_ctx_config.ciphersuites); - server.tls_ctx_config.ciphersuites = zstrdup(o->ptr); - } config_set_special_field("tls-prefer-server-ciphers") { - redisTLSContextConfig tmpctx = server.tls_ctx_config; - tmpctx.prefer_server_ciphers = yesnotoi(o->ptr); - if (tlsConfigure(&tmpctx) == C_ERR) { - addReplyError(c, "Unable to reconfigure TLS. Check server logs."); - return; - } - server.tls_ctx_config.prefer_server_ciphers = tmpctx.prefer_server_ciphers; -#endif /* USE_OPENSSL */ /* Everyhing else is an error... */ } config_set_else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", @@ -1084,51 +797,14 @@ void configGetCommand(client *c) { } /* String values */ - config_get_string_field("dbfilename",server.rdb_filename); config_get_string_field("logfile",server.logfile); -#ifdef USE_OPENSSL - config_get_string_field("tls-cert-file",server.tls_ctx_config.cert_file); - config_get_string_field("tls-key-file",server.tls_ctx_config.key_file); - config_get_string_field("tls-dh-params-file",server.tls_ctx_config.dh_params_file); - config_get_string_field("tls-ca-cert-file",server.tls_ctx_config.ca_cert_file); - config_get_string_field("tls-ca-cert-dir",server.tls_ctx_config.ca_cert_dir); - config_get_string_field("tls-protocols",server.tls_ctx_config.protocols); - config_get_string_field("tls-ciphers",server.tls_ctx_config.ciphers); - config_get_string_field("tls-ciphersuites",server.tls_ctx_config.ciphersuites); -#endif /* Numerical values */ - config_get_numerical_field("maxmemory",server.maxmemory); config_get_numerical_field("client-query-buffer-limit",server.client_max_querybuf_len); - config_get_numerical_field("auto-aof-rewrite-min-size", - server.aof_rewrite_min_size); - config_get_numerical_field("stream-node-max-entries", - server.stream_node_max_entries); - config_get_numerical_field("tls-port",server.tls_port); - config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); - config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); - config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); - config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); - config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("hz",server.config_hz); - - /* Bool (yes/no) values */ - config_get_bool_field("tls-cluster",server.tls_cluster); - config_get_bool_field("tls-replication",server.tls_replication); - config_get_bool_field("tls-auth-clients",server.tls_auth_clients); - config_get_bool_field("tls-prefer-server-ciphers", - server.tls_ctx_config.prefer_server_ciphers); /* Everything we can't handle with macros follows. */ - if (stringmatch(pattern,"appendonly",1)) { - addReplyBulkCString(c,"appendonly"); - addReplyBulkCString(c,server.aof_enabled ? "yes" : "no"); - matches++; - } if (stringmatch(pattern,"dir",1)) { char buf[1024]; @@ -1818,43 +1494,15 @@ int rewriteConfig(char *path) { rewriteConfigBindOption(state); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); - rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); - rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,CONFIG_DEFAULT_SYSLOG_IDENT); rewriteConfigSaveOption(state); rewriteConfigUserOption(state); - rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state,"replicaof"); - rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); - rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); rewriteConfigRequirepassOption(state,"requirepass"); - rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); - rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); rewriteConfigBytesOption(state,"client-query-buffer-limit",server.client_max_querybuf_len,PROTO_MAX_QUERYBUF_LEN); - rewriteConfigYesNoOption(state,"appendonly",server.aof_enabled,0); - rewriteConfigStringOption(state,"appendfilename",server.aof_filename,CONFIG_DEFAULT_AOF_FILENAME); - rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,AOF_REWRITE_MIN_SIZE); - rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigNotifykeyspaceeventsOption(state); - rewriteConfigNumericalOption(state,"stream-node-max-entries",server.stream_node_max_entries,OBJ_STREAM_NODE_MAX_ENTRIES); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); - -#ifdef USE_OPENSSL - rewriteConfigYesNoOption(state,"tls-cluster",server.tls_cluster,0); - rewriteConfigYesNoOption(state,"tls-replication",server.tls_replication,0); - rewriteConfigYesNoOption(state,"tls-auth-clients",server.tls_auth_clients,1); - rewriteConfigStringOption(state,"tls-cert-file",server.tls_ctx_config.cert_file,NULL); - rewriteConfigStringOption(state,"tls-key-file",server.tls_ctx_config.key_file,NULL); - rewriteConfigStringOption(state,"tls-dh-params-file",server.tls_ctx_config.dh_params_file,NULL); - rewriteConfigStringOption(state,"tls-ca-cert-file",server.tls_ctx_config.ca_cert_file,NULL); - rewriteConfigStringOption(state,"tls-ca-cert-dir",server.tls_ctx_config.ca_cert_dir,NULL); - rewriteConfigStringOption(state,"tls-protocols",server.tls_ctx_config.protocols,NULL); - rewriteConfigStringOption(state,"tls-ciphers",server.tls_ctx_config.ciphers,NULL); - rewriteConfigStringOption(state,"tls-ciphersuites",server.tls_ctx_config.ciphersuites,NULL); - rewriteConfigYesNoOption(state,"tls-prefer-server-ciphers",server.tls_ctx_config.prefer_server_ciphers,0); -#endif /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); @@ -2385,6 +2033,34 @@ static int isValidActiveDefrag(int val, char **err) { return 1; } +static int isValidDBfilename(char *val, char **err) { + if (!pathIsBaseName(val)) { + *err = "dbfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int isValidAOFfilename(char *val, char **err) { + if (!pathIsBaseName(val)) { + *err = "appendfilename can't be a path, just a filename"; + return 0; + } + return 1; +} + +static int updateHZ(long long val, long long prev, char **err) { + UNUSED(prev); + UNUSED(err); + /* Hz is more an hint from the user, so we accept values out of range + * but cap them to reasonable values. */ + server.config_hz = val; + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; + server.hz = server.config_hz; + return 1; +} + static int updateJemallocBgThread(int val, int prev, char **err) { UNUSED(prev); UNUSED(err); @@ -2392,106 +2068,227 @@ static int updateJemallocBgThread(int val, int prev, char **err) { return 1; } +static int updateReplBacklogSize(long long val, long long prev, char **err) { + /* resizeReplicationBacklog sets server.repl_backlog_size, and relies on + * being able to tell when the size changes, so restore prev becore calling it. */ + UNUSED(err); + server.repl_backlog_size = prev; + resizeReplicationBacklog(val); + return 1; +} + +static int updateMaxmemory(long long val, long long prev, char **err) { + UNUSED(prev); + UNUSED(err); + if (val) { + if ((unsigned long long)val < zmalloc_used_memory()) { + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); + } + freeMemoryIfNeededAndSafe(); + } + return 1; +} + +static int updateGoodSlaves(long long val, long long prev, char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + refreshGoodSlavesCount(); + return 1; +} + +static int updateAppendonly(int val, int prev, char **err) { + UNUSED(prev); + if (val == 0 && server.aof_state != AOF_OFF) { + stopAppendOnly(); + } else if (val && server.aof_state == AOF_OFF) { + if (startAppendOnly() == C_ERR) { + *err = "Unable to turn on AOF. Check server logs."; + return 0; + } + } + return 1; +} + +static int updateMaxclients(long long val, long long prev, char **err) { + /* Try to check if the OS is capable of supporting so many FDs. */ + if (val > prev) { + adjustOpenFilesLimit(); + if (server.maxclients != val) { + static char msg[128]; + sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); + *err = msg; + return 0; + } + if ((unsigned int) aeGetSetSize(server.el) < + server.maxclients + CONFIG_FDSET_INCR) + { + if (aeResizeSetSize(server.el, + server.maxclients + CONFIG_FDSET_INCR) == AE_ERR) + { + *err = "The event loop API used by Redis is not able to handle the specified number of clients"; + return 0; + } + } + } + return 1; +} + +#ifdef USE_OPENSSL +static int updateTlsCfg(char *val, char *prev, char **err) { + UNUSED(val); + UNUSED(prev); + UNUSED(err); + if (tlsConfigure(&server.tls_ctx_config) == C_ERR) { + *err = "Unable to configure tls-cert-file. Check server logs."; + return 0; + } + return 1; +} +static int updateTlsCfgBool(int val, int prev, char **err) { + UNUSED(val); + UNUSED(prev); + return updateTlsCfg(NULL, NULL, err); +} +#endif /* USE_OPENSSL */ + standardConfig configs[] = { /* Bool configs */ - createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, CONFIG_DEFAULT_RDB_CHECKSUM ,NULL, NULL), + createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL), createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0 ,NULL, NULL), - createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, CONFIG_DEFAULT_IO_THREADS_DO_READS ,NULL, NULL), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0 ,NULL, NULL), /* Read + parse from threads? */ createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1 ,NULL, NULL), - createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, CONFIG_DEFAULT_ALWAYS_SHOW_LOGO ,NULL, NULL), - createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, CONFIG_DEFAULT_PROTECTED_MODE ,NULL, NULL), - createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, CONFIG_DEFAULT_RDB_COMPRESSION ,NULL, NULL), - createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, CONFIG_DEFAULT_ACTIVE_REHASHING ,NULL, NULL), - createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR ,NULL, NULL), - createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, CONFIG_DEFAULT_DYNAMIC_HZ ,NULL, NULL), - createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION ,NULL, NULL), - createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE ,NULL, NULL), - createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL ,NULL, NULL), - createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY ,NULL, NULL), - createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, CONFIG_DEFAULT_REPL_DISKLESS_SYNC ,NULL, NULL), - createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, CONFIG_DEFAULT_GOPHER_ENABLED ,NULL, NULL), - createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC ,NULL, NULL), - createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE ,NULL, NULL), - createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE ,NULL, NULL), - createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC ,NULL, NULL), - createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, CONFIG_DEFAULT_AOF_LOAD_TRUNCATED ,NULL, NULL), - createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE ,NULL, NULL), - createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, CLUSTER_DEFAULT_SLAVE_NO_FAILOVER ,NULL, NULL), - createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, CONFIG_DEFAULT_SLAVE_LAZY_FLUSH ,NULL, NULL), - createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA ,NULL, NULL), - createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, CONFIG_DEFAULT_SLAVE_READ_ONLY ,NULL, NULL), - createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY ,NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0 ,NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1 ,NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1 ,NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1 ,NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1 ,NULL, NULL), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1 ,NULL, NULL), /* Adapt hz to # of clients.*/ + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0 ,NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0 ,NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0 ,NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0 ,NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0 ,NULL, NULL), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0 ,NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1 ,NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0 ,NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1 ,NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, 1 ,NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1 ,NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1 ,NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, 0 ,NULL, NULL), /* Failover by default. */ + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, 0 ,NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1 ,NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, 1 ,NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1 ,NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), + createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), + createBoolConfig("cluster-enabled", NULL, IMMUTABLE_CONFIG, server.cluster_enabled, 0, NULL, NULL), + createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, server.aof_enabled, 0, NULL, updateAppendonly), /* String Configs */ - createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, CONFIG_DEFAULT_ACL_FILENAME ,NULL, NULL), + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "" ,NULL, NULL), createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL ,NULL, NULL), - createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, CONFIG_DEFAULT_PID_FILE ,NULL, NULL), - createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP ,NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL ,NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL ,NULL, NULL), createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL ,NULL, NULL), createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL ,NULL, NULL), createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL ,NULL, NULL), + createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), + createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), + createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), /* Enum Configs */ createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE ,NULL, NULL), createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0 ,NULL, NULL), - createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, CONFIG_DEFAULT_REPL_DISKLESS_LOAD ,NULL, NULL), - createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, CONFIG_DEFAULT_VERBOSITY ,NULL, NULL), - createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, CONFIG_DEFAULT_MAXMEMORY_POLICY ,NULL, NULL), - createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, CONFIG_DEFAULT_AOF_FSYNC ,NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED ,NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE ,NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION ,NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC ,NULL, NULL), /* Integer configs */ - createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, CONFIG_DEFAULT_DBNUM, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, CONFIG_DEFAULT_SERVER_PORT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, CONFIG_DEFAULT_IO_THREADS_NUM, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, AOF_REWRITE_PERC, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, CLUSTER_DEFAULT_SLAVE_VALIDITY, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, OBJ_LIST_MAX_ZIPLIST_SIZE, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, CONFIG_DEFAULT_TCP_KEEPALIVE, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, CLUSTER_DEFAULT_MIGRATION_BARRIER, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, CONFIG_DEFAULT_DEFRAG_CYCLE_MIN, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, CONFIG_DEFAULT_DEFRAG_CYCLE_MAX, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, CONFIG_DEFAULT_LFU_LOG_FACTOR, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, CONFIG_DEFAULT_LFU_DECAY_TIME, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, CONFIG_DEFAULT_SLAVE_PRIORITY, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, CONFIG_DEFAULT_MAXMEMORY_SAMPLES, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, CONFIG_DEFAULT_CLIENT_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, CONFIG_DEFAULT_TCP_BACKLOG, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, CONFIG_DEFAULT_REPL_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, OBJ_LIST_COMPRESS_DEPTH, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, CONFIG_DEFAULT_KEY_LOAD_DELAY, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG ,NULL, NULL), /* TCP port. */ + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, 1, INTEGER_CONFIG ,NULL, NULL), /* Single threaded by default */ + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG ,NULL, NULL), /* Slave max data age factor. */ + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG ,NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, 25, INTEGER_CONFIG ,NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG ,NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG ,NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, 10, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, 1, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG ,NULL, NULL), /* Default client timeout: infinite */ + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG ,NULL, NULL), /* TCP listen backlog. */ + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG ,NULL, NULL), /* Default: Use +10000 offset. */ + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG ,NULL, NULL), /* Use server.port */ + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG ,NULL, NULL), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG ,NULL, NULL), /* Default: 10% tracking table max fill. */ + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG ,NULL, NULL), /* From 1 to 10. */ + createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG ,NULL, updateHZ), + createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG ,NULL, updateGoodSlaves), + createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG ,NULL, updateGoodSlaves), + + /* Unsigned int configs */ + createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), /* Unsigned Long configs */ - createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS, INTEGER_CONFIG ,NULL, NULL), - createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, CONFIG_DEFAULT_SLOWLOG_MAX_LEN, INTEGER_CONFIG ,NULL, NULL), + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG ,NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG ,NULL, NULL), /* Long Long configs */ - createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, LUA_SCRIPT_TIME_LIMIT, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, CLUSTER_DEFAULT_NODE_TIMEOUT, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, CONFIG_DEFAULT_PROTO_MAX_BULK_LEN, MEMORY_CONFIG ,NULL, NULL), + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG ,NULL, NULL),/* milliseconds */ + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG ,NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ + + /* Unsigned Long Long configs */ + createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG ,NULL, updateMaxmemory), /* Size_t configs */ - createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, OBJ_HASH_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, OBJ_SET_MAX_INTSET_ENTRIES, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, OBJ_ZSET_MAX_ZIPLIST_ENTRIES, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, OBJ_HASH_MAX_ZIPLIST_VALUE, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, OBJ_STREAM_NODE_MAX_BYTES, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, OBJ_ZSET_MAX_ZIPLIST_VALUE, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG ,NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG ,NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG ,NULL, NULL), + + /* Other configs */ + createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60 ,INTEGER_CONFIG ,NULL, NULL), /* Default: 1 hour */ + createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG ,NULL, NULL), + +#ifdef USE_OPENSSL + createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG ,NULL, NULL), /* TCP port. */ + createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL), + createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL), + createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL), + createBoolConfig("tls-prefer-server-ciphers", NULL, MODIFIABLE_CONFIG, server.tls_ctx_config.prefer_server_ciphers, 0, NULL, updateTlsCfgBool), + createStringConfig("tls-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-key-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.key_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-dh-params-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.dh_params_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-file", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_file, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ca-cert-dir", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ca_cert_dir, NULL, NULL, updateTlsCfg), + createStringConfig("tls-protocols", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.protocols, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphers", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphers, NULL, NULL, updateTlsCfg), + createStringConfig("tls-ciphersuites", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.tls_ctx_config.ciphersuites, NULL, NULL, updateTlsCfg), +#endif /* NULL Terminator */ {NULL} diff --git a/src/server.c b/src/server.c index 497d093f1..72f136446 100644 --- a/src/server.c +++ b/src/server.c @@ -2274,49 +2274,18 @@ void initServerConfig(void) { server.timezone = getTimeZone(); /* Initialized by tzset(). */ server.configfile = NULL; server.executable = NULL; - server.hz = server.config_hz = CONFIG_DEFAULT_HZ; - server.dynamic_hz = CONFIG_DEFAULT_DYNAMIC_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; - server.port = CONFIG_DEFAULT_SERVER_PORT; - server.tls_port = CONFIG_DEFAULT_SERVER_TLS_PORT; - server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; server.bindaddr_count = 0; - server.unixsocket = NULL; server.unixsocketperm = CONFIG_DEFAULT_UNIX_SOCKET_PERM; server.ipfd_count = 0; server.tlsfd_count = 0; server.sofd = -1; - server.protected_mode = CONFIG_DEFAULT_PROTECTED_MODE; - server.gopher_enabled = CONFIG_DEFAULT_GOPHER_ENABLED; - server.dbnum = CONFIG_DEFAULT_DBNUM; - server.verbosity = CONFIG_DEFAULT_VERBOSITY; - server.maxidletime = CONFIG_DEFAULT_CLIENT_TIMEOUT; - server.tcpkeepalive = CONFIG_DEFAULT_TCP_KEEPALIVE; server.active_expire_enabled = 1; - server.active_expire_effort = CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT; - server.jemalloc_bg_thread = 1; - server.active_defrag_ignore_bytes = CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES; - server.active_defrag_threshold_lower = CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER; - server.active_defrag_threshold_upper = CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER; - server.active_defrag_cycle_min = CONFIG_DEFAULT_DEFRAG_CYCLE_MIN; - server.active_defrag_cycle_max = CONFIG_DEFAULT_DEFRAG_CYCLE_MAX; - server.active_defrag_max_scan_fields = CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS; - server.proto_max_bulk_len = CONFIG_DEFAULT_PROTO_MAX_BULK_LEN; server.client_max_querybuf_len = PROTO_MAX_QUERYBUF_LEN; server.saveparams = NULL; server.loading = 0; server.logfile = zstrdup(CONFIG_DEFAULT_LOGFILE); - server.syslog_enabled = CONFIG_DEFAULT_SYSLOG_ENABLED; - server.syslog_ident = zstrdup(CONFIG_DEFAULT_SYSLOG_IDENT); - server.syslog_facility = LOG_LOCAL0; - server.daemonize = CONFIG_DEFAULT_DAEMONIZE; - server.supervised = 0; - server.supervised_mode = SUPERVISED_NONE; server.aof_state = AOF_OFF; - server.aof_fsync = CONFIG_DEFAULT_AOF_FSYNC; - server.aof_no_fsync_on_rewrite = CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE; - server.aof_rewrite_perc = AOF_REWRITE_PERC; - server.aof_rewrite_min_size = AOF_REWRITE_MIN_SIZE; server.aof_rewrite_base_size = 0; server.aof_rewrite_scheduled = 0; server.aof_flush_sleep = 0; @@ -2328,63 +2297,18 @@ void initServerConfig(void) { server.aof_fd = -1; server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; - server.aof_rewrite_incremental_fsync = CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC; - server.rdb_save_incremental_fsync = CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC; - server.rdb_key_save_delay = CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY; - server.key_load_delay = CONFIG_DEFAULT_KEY_LOAD_DELAY; - server.aof_load_truncated = CONFIG_DEFAULT_AOF_LOAD_TRUNCATED; - server.aof_use_rdb_preamble = CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE; server.pidfile = NULL; - server.rdb_filename = zstrdup(CONFIG_DEFAULT_RDB_FILENAME); - server.aof_filename = zstrdup(CONFIG_DEFAULT_AOF_FILENAME); - server.acl_filename = zstrdup(CONFIG_DEFAULT_ACL_FILENAME); - server.rdb_compression = CONFIG_DEFAULT_RDB_COMPRESSION; - server.rdb_checksum = CONFIG_DEFAULT_RDB_CHECKSUM; - server.stop_writes_on_bgsave_err = CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR; - server.activerehashing = CONFIG_DEFAULT_ACTIVE_REHASHING; server.active_defrag_running = 0; server.notify_keyspace_events = 0; - server.maxclients = CONFIG_DEFAULT_MAX_CLIENTS; server.blocked_clients = 0; memset(server.blocked_clients_by_type,0, sizeof(server.blocked_clients_by_type)); - server.maxmemory = CONFIG_DEFAULT_MAXMEMORY; - server.maxmemory_policy = CONFIG_DEFAULT_MAXMEMORY_POLICY; - server.maxmemory_samples = CONFIG_DEFAULT_MAXMEMORY_SAMPLES; - server.lfu_log_factor = CONFIG_DEFAULT_LFU_LOG_FACTOR; - server.lfu_decay_time = CONFIG_DEFAULT_LFU_DECAY_TIME; - server.hash_max_ziplist_entries = OBJ_HASH_MAX_ZIPLIST_ENTRIES; - server.hash_max_ziplist_value = OBJ_HASH_MAX_ZIPLIST_VALUE; - server.list_max_ziplist_size = OBJ_LIST_MAX_ZIPLIST_SIZE; - server.list_compress_depth = OBJ_LIST_COMPRESS_DEPTH; - server.set_max_intset_entries = OBJ_SET_MAX_INTSET_ENTRIES; - server.zset_max_ziplist_entries = OBJ_ZSET_MAX_ZIPLIST_ENTRIES; - server.zset_max_ziplist_value = OBJ_ZSET_MAX_ZIPLIST_VALUE; - server.hll_sparse_max_bytes = CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES; - server.stream_node_max_bytes = OBJ_STREAM_NODE_MAX_BYTES; - server.stream_node_max_entries = OBJ_STREAM_NODE_MAX_ENTRIES; server.shutdown_asap = 0; - server.cluster_enabled = 0; - server.cluster_node_timeout = CLUSTER_DEFAULT_NODE_TIMEOUT; - server.cluster_migration_barrier = CLUSTER_DEFAULT_MIGRATION_BARRIER; - server.cluster_slave_validity_factor = CLUSTER_DEFAULT_SLAVE_VALIDITY; - server.cluster_require_full_coverage = CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE; - server.cluster_slave_no_failover = CLUSTER_DEFAULT_SLAVE_NO_FAILOVER; server.cluster_configfile = zstrdup(CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); - server.cluster_announce_ip = CONFIG_DEFAULT_CLUSTER_ANNOUNCE_IP; - server.cluster_announce_port = CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT; - server.cluster_announce_bus_port = CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT; server.cluster_module_flags = CLUSTER_MODULE_FLAG_NONE; server.migrate_cached_sockets = dictCreate(&migrateCacheDictType,NULL); server.next_client_id = 1; /* Client IDs, start from 1 .*/ server.loading_process_events_interval_bytes = (1024*1024*2); - server.lazyfree_lazy_eviction = CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION; - server.lazyfree_lazy_expire = CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE; - server.lazyfree_lazy_server_del = CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL; - server.always_show_logo = CONFIG_DEFAULT_ALWAYS_SHOW_LOGO; - server.lua_time_limit = LUA_SCRIPT_TIME_LIMIT; - server.io_threads_num = CONFIG_DEFAULT_IO_THREADS_NUM; - server.io_threads_do_reads = CONFIG_DEFAULT_IO_THREADS_DO_READS; server.lruclock = getLRUClock(); resetServerSaveParams(); @@ -2405,31 +2329,14 @@ void initServerConfig(void) { server.repl_transfer_fd = -1; server.repl_transfer_s = NULL; server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; - server.repl_serve_stale_data = CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA; - server.repl_slave_ro = CONFIG_DEFAULT_SLAVE_READ_ONLY; - server.repl_slave_ignore_maxmemory = CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY; - server.repl_slave_lazy_flush = CONFIG_DEFAULT_SLAVE_LAZY_FLUSH; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ - server.repl_disable_tcp_nodelay = CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY; - server.repl_diskless_sync = CONFIG_DEFAULT_REPL_DISKLESS_SYNC; - server.repl_diskless_load = CONFIG_DEFAULT_REPL_DISKLESS_LOAD; - server.repl_diskless_sync_delay = CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY; - server.repl_ping_slave_period = CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD; - server.repl_timeout = CONFIG_DEFAULT_REPL_TIMEOUT; - server.repl_min_slaves_to_write = CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE; - server.repl_min_slaves_max_lag = CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG; - server.slave_priority = CONFIG_DEFAULT_SLAVE_PRIORITY; - server.slave_announce_ip = CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP; - server.slave_announce_port = CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT; server.master_repl_offset = 0; /* Replication partial resync backlog */ server.repl_backlog = NULL; - server.repl_backlog_size = CONFIG_DEFAULT_REPL_BACKLOG_SIZE; server.repl_backlog_histlen = 0; server.repl_backlog_idx = 0; server.repl_backlog_off = 0; - server.repl_backlog_time_limit = CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT; server.repl_no_slaves_since = time(NULL); /* Client output buffer limits */ @@ -2462,16 +2369,6 @@ void initServerConfig(void) { server.xclaimCommand = lookupCommandByCString("xclaim"); server.xgroupCommand = lookupCommandByCString("xgroup"); - /* Slow log */ - server.slowlog_log_slower_than = CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN; - server.slowlog_max_len = CONFIG_DEFAULT_SLOWLOG_MAX_LEN; - - /* Latency monitor */ - server.latency_monitor_threshold = CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD; - - /* Tracking. */ - server.tracking_table_max_fill = CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL; - /* Debugging */ server.assert_failed = ""; server.assert_file = ""; @@ -2785,6 +2682,8 @@ void initServer(void) { server.syslog_facility); } + /* Initialization after setting defaults from the config system. */ + server.aof_state = server.aof_enabled ? AOF_ON : AOF_OFF; server.hz = server.config_hz; server.pid = getpid(); server.current_client = NULL; diff --git a/src/server.h b/src/server.h index f31cb8fc1..d251fa026 100644 --- a/src/server.h +++ b/src/server.h @@ -84,18 +84,10 @@ typedef long long ustime_t; /* microsecond time type. */ #define C_ERR -1 /* Static server configuration */ -#define CONFIG_DEFAULT_DYNAMIC_HZ 1 /* Adapt hz to # of clients.*/ #define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 #define CONFIG_MAX_HZ 500 #define MAX_CLIENTS_PER_CLOCK_TICK 200 /* HZ is adapted based on that. */ -#define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port. */ -#define CONFIG_DEFAULT_SERVER_TLS_PORT 0 /* TCP port. */ -#define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog. */ -#define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* Default client timeout: infinite */ -#define CONFIG_DEFAULT_DBNUM 16 -#define CONFIG_DEFAULT_IO_THREADS_NUM 1 /* Single threaded by default */ -#define CONFIG_DEFAULT_IO_THREADS_DO_READS 0 /* Read + parse from threads? */ #define CONFIG_MAX_LINE 1024 #define CRON_DBS_PER_CALL 16 #define NET_MAX_WRITES_PER_EVENT (1024*64) @@ -103,83 +95,21 @@ typedef long long ustime_t; /* microsecond time type. */ #define OBJ_SHARED_INTEGERS 10000 #define OBJ_SHARED_BULKHDR_LEN 32 #define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages.*/ -#define AOF_REWRITE_PERC 100 -#define AOF_REWRITE_MIN_SIZE (64*1024*1024) #define AOF_REWRITE_ITEMS_PER_CMD 64 #define AOF_READ_DIFF_INTERVAL_BYTES (1024*10) -#define CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN 10000 -#define CONFIG_DEFAULT_SLOWLOG_MAX_LEN 128 -#define CONFIG_DEFAULT_MAX_CLIENTS 10000 #define CONFIG_AUTHPASS_MAX_LEN 512 -#define CONFIG_DEFAULT_SLAVE_PRIORITY 100 -#define CONFIG_DEFAULT_REPL_TIMEOUT 60 -#define CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD 10 #define CONFIG_RUN_ID_SIZE 40 #define RDB_EOF_MARK_SIZE 40 -#define CONFIG_DEFAULT_REPL_BACKLOG_SIZE (1024*1024) /* 1mb */ -#define CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT (60*60) /* 1 hour */ #define CONFIG_REPL_BACKLOG_MIN_SIZE (1024*16) /* 16k */ #define CONFIG_BGSAVE_RETRY_DELAY 5 /* Wait a few secs before trying again. */ #define CONFIG_DEFAULT_PID_FILE "/var/run/redis.pid" -#define CONFIG_DEFAULT_SYSLOG_IDENT "redis" #define CONFIG_DEFAULT_CLUSTER_CONFIG_FILE "nodes.conf" -#define CONFIG_DEFAULT_CLUSTER_ANNOUNCE_IP NULL /* Auto detect. */ -#define CONFIG_DEFAULT_CLUSTER_ANNOUNCE_PORT 0 /* Use server.port */ -#define CONFIG_DEFAULT_CLUSTER_ANNOUNCE_BUS_PORT 0 /* Use +10000 offset. */ -#define CONFIG_DEFAULT_DAEMONIZE 0 #define CONFIG_DEFAULT_UNIX_SOCKET_PERM 0 -#define CONFIG_DEFAULT_TCP_KEEPALIVE 300 -#define CONFIG_DEFAULT_PROTECTED_MODE 1 -#define CONFIG_DEFAULT_GOPHER_ENABLED 0 #define CONFIG_DEFAULT_LOGFILE "" -#define CONFIG_DEFAULT_SYSLOG_ENABLED 0 -#define CONFIG_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR 1 -#define CONFIG_DEFAULT_RDB_COMPRESSION 1 -#define CONFIG_DEFAULT_RDB_CHECKSUM 1 -#define CONFIG_DEFAULT_RDB_FILENAME "dump.rdb" -#define CONFIG_DEFAULT_REPL_DISKLESS_SYNC 0 -#define CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY 5 -#define CONFIG_DEFAULT_RDB_KEY_SAVE_DELAY 0 -#define CONFIG_DEFAULT_KEY_LOAD_DELAY 0 -#define CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA 1 -#define CONFIG_DEFAULT_SLAVE_READ_ONLY 1 -#define CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY 1 -#define CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP NULL -#define CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT 0 -#define CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 -#define CONFIG_DEFAULT_MAXMEMORY 0 -#define CONFIG_DEFAULT_MAXMEMORY_SAMPLES 5 -#define CONFIG_DEFAULT_LFU_LOG_FACTOR 10 -#define CONFIG_DEFAULT_LFU_DECAY_TIME 1 -#define CONFIG_DEFAULT_AOF_FILENAME "appendonly.aof" -#define CONFIG_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0 -#define CONFIG_DEFAULT_AOF_LOAD_TRUNCATED 1 -#define CONFIG_DEFAULT_AOF_USE_RDB_PREAMBLE 1 -#define CONFIG_DEFAULT_ACTIVE_REHASHING 1 -#define CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 -#define CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC 1 -#define CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE 0 -#define CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG 10 -#define CONFIG_DEFAULT_ACL_FILENAME "" #define NET_IP_STR_LEN 46 /* INET6_ADDRSTRLEN is 46, but we need to be sure */ #define NET_PEER_ID_LEN (NET_IP_STR_LEN+32) /* Must be enough for ip:port */ #define CONFIG_BINDADDR_MAX 16 #define CONFIG_MIN_RESERVED_FDS 32 -#define CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD 0 -#define CONFIG_DEFAULT_SLAVE_LAZY_FLUSH 0 -#define CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION 0 -#define CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE 0 -#define CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL 0 -#define CONFIG_DEFAULT_ALWAYS_SHOW_LOGO 0 -#define CONFIG_DEFAULT_DEFRAG_THRESHOLD_LOWER 10 /* don't defrag when fragmentation is below 10% */ -#define CONFIG_DEFAULT_DEFRAG_THRESHOLD_UPPER 100 /* maximum defrag force at 100% fragmentation */ -#define CONFIG_DEFAULT_DEFRAG_IGNORE_BYTES (100<<20) /* don't defrag if frag overhead is below 100mb */ -#define CONFIG_DEFAULT_DEFRAG_CYCLE_MIN 1 /* 1% CPU min (at lower threshold) */ -#define CONFIG_DEFAULT_DEFRAG_CYCLE_MAX 25 /* 25% CPU max (at upper threshold) */ -#define CONFIG_DEFAULT_DEFRAG_MAX_SCAN_FIELDS 1000 /* keys with more than 1000 fields will be processed separately */ -#define CONFIG_DEFAULT_PROTO_MAX_BULK_LEN (512ll*1024*1024) /* Bulk request max size */ -#define CONFIG_DEFAULT_TRACKING_TABLE_MAX_FILL 10 /* 10% tracking table max fill. */ -#define CONFIG_DEFAULT_ACTIVE_EXPIRE_EFFORT 1 /* From 1 to 10. */ #define ACTIVE_EXPIRE_CYCLE_SLOW 0 #define ACTIVE_EXPIRE_CYCLE_FAST 1 @@ -390,7 +320,6 @@ typedef long long ustime_t; /* microsecond time type. */ #define LL_NOTICE 2 #define LL_WARNING 3 #define LL_RAW (1<<10) /* Modifier to log without timestamp */ -#define CONFIG_DEFAULT_VERBOSITY LL_NOTICE /* Supervision options */ #define SUPERVISED_NONE 0 @@ -408,29 +337,11 @@ typedef long long ustime_t; /* microsecond time type. */ #define AOF_FSYNC_NO 0 #define AOF_FSYNC_ALWAYS 1 #define AOF_FSYNC_EVERYSEC 2 -#define CONFIG_DEFAULT_AOF_FSYNC AOF_FSYNC_EVERYSEC /* Replication diskless load defines */ #define REPL_DISKLESS_LOAD_DISABLED 0 #define REPL_DISKLESS_LOAD_WHEN_DB_EMPTY 1 #define REPL_DISKLESS_LOAD_SWAPDB 2 -#define CONFIG_DEFAULT_REPL_DISKLESS_LOAD REPL_DISKLESS_LOAD_DISABLED - -/* Zipped structures related defaults */ -#define OBJ_HASH_MAX_ZIPLIST_ENTRIES 512 -#define OBJ_HASH_MAX_ZIPLIST_VALUE 64 -#define OBJ_SET_MAX_INTSET_ENTRIES 512 -#define OBJ_ZSET_MAX_ZIPLIST_ENTRIES 128 -#define OBJ_ZSET_MAX_ZIPLIST_VALUE 64 -#define OBJ_STREAM_NODE_MAX_BYTES 4096 -#define OBJ_STREAM_NODE_MAX_ENTRIES 100 - -/* List defaults */ -#define OBJ_LIST_MAX_ZIPLIST_SIZE -2 -#define OBJ_LIST_COMPRESS_DEPTH 0 - -/* HyperLogLog defines */ -#define CONFIG_DEFAULT_HLL_SPARSE_MAX_BYTES 3000 /* Sets operations codes */ #define SET_OP_UNION 0 @@ -455,11 +366,6 @@ typedef long long ustime_t; /* microsecond time type. */ #define MAXMEMORY_ALLKEYS_RANDOM ((6<<8)|MAXMEMORY_FLAG_ALLKEYS) #define MAXMEMORY_NO_EVICTION (7<<8) -#define CONFIG_DEFAULT_MAXMEMORY_POLICY MAXMEMORY_NO_EVICTION - -/* Scripting */ -#define LUA_SCRIPT_TIME_LIMIT 5000 /* milliseconds */ - /* Units */ #define UNIT_SECONDS 0 #define UNIT_MILLISECONDS 1 @@ -1395,7 +1301,7 @@ struct redisServer { size_t zset_max_ziplist_value; size_t hll_sparse_max_bytes; size_t stream_node_max_bytes; - int64_t stream_node_max_entries; + long long stream_node_max_entries; /* List parameters */ int list_max_ziplist_size; int list_compress_depth; diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 2581eb83d..3bf3d367b 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -57,4 +57,69 @@ start_server {tags {"introspection"}} { fail "Client still listed in CLIENT LIST after SETNAME." } } + + test {CONFIG sanity} { + # Do CONFIG GET, CONFIG SET and then CONFIG GET again + # Skip immutable configs, one with no get, and other complicated configs + set skip_configs { + rdbchecksum + daemonize + io-threads-do-reads + lua-replicate-commands + always-show-logo + syslog-enabled + cluster-enabled + aclfile + unixsocket + pidfile + syslog-ident + appendfilename + supervised + syslog-facility + databases + port + io-threads + tls-port + tls-prefer-server-ciphers + tls-cert-file + tls-key-file + tls-dh-params-file + tls-ca-cert-file + tls-ca-cert-dir + tls-protocols + tls-ciphers + tls-ciphersuites + logfile + unixsocketperm + slaveof + bind + requirepass + } + + set configs {} + foreach {k v} [r config get *] { + if {[lsearch $skip_configs $k] != -1} { + continue + } + dict set configs $k $v + # try to set the config to the same value it already has + r config set $k $v + } + + set newconfigs {} + foreach {k v} [r config get *] { + if {[lsearch $skip_configs $k] != -1} { + continue + } + dict set newconfigs $k $v + } + + dict for {k v} $configs { + set vv [dict get $newconfigs $k] + if {$v != $vv} { + fail "config $k mismatch, expecting $v but got $vv" + } + + } + } } From 6c3e6b09703fba779466f11c3fd8e5674e19cef3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 29 Nov 2019 17:35:59 +0100 Subject: [PATCH 306/320] Fix Pi build needing -latomic. Issue #6275. --- src/Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Makefile b/src/Makefile index 9fc230f94..e67731982 100644 --- a/src/Makefile +++ b/src/Makefile @@ -77,6 +77,15 @@ FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb +# Linux ARM needs -latomic at linking time +ifneq (,$(filter aarch64 armv,$(uname_M))) + FINAL_LIBS+=-latomic +else +ifneq (,$(findstring armv,$(uname_M))) + FINAL_LIBS+=-latomic +endif +endif + ifeq ($(uname_S),SunOS) # SunOS ifneq ($(@@),32bit) From 015136bf06c991a750087afa567322daf23ce52b Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 1 Dec 2019 08:19:25 +0200 Subject: [PATCH 307/320] config.c post refactory cleanup --- src/config.c | 270 ++++++++++++++++++++++++--------------------------- 1 file changed, 125 insertions(+), 145 deletions(-) diff --git a/src/config.c b/src/config.c index 16cc74f57..5bfb0de5c 100644 --- a/src/config.c +++ b/src/config.c @@ -1802,6 +1802,36 @@ static void numericConfigInit(typeData data) { SET_NUMERIC_TYPE(data.numeric.default_value) } +static int numericBoundaryCheck(typeData data, long long ll, char **err) { + if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || + data.numeric.numeric_type == NUMERIC_TYPE_UINT || + data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { + /* Boundary check for unsigned types */ + unsigned long long ull = ll; + unsigned long long upper_bound = data.numeric.upper_bound; + unsigned long long lower_bound = data.numeric.lower_bound; + if (ull > upper_bound || ull < lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %llu and %llu inclusive", + lower_bound, + upper_bound); + *err = loadbuf; + return 0; + } + } else { + /* Boundary check for signed types */ + if (ll > data.numeric.upper_bound || ll < data.numeric.lower_bound) { + snprintf(loadbuf, LOADBUF_SIZE, + "argument must be between %lld and %lld inclusive", + data.numeric.lower_bound, + data.numeric.upper_bound); + *err = loadbuf; + return 0; + } + } + return 1; +} + static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { long long ll; @@ -1824,33 +1854,8 @@ static int numericConfigLoad(typeData data, sds *argv, int argc, char **err) { } } - if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || - data.numeric.numeric_type == NUMERIC_TYPE_UINT || - data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - /* Boundary check for unsigned types */ - unsigned long long ull = ll; - unsigned long long upper_bound = data.numeric.upper_bound; - unsigned long long lower_bound = data.numeric.lower_bound; - if (ull > upper_bound || ull < lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %llu and %llu inclusive", - lower_bound, - upper_bound); - *err = loadbuf; - return 0; - } - } else { - /* Boundary check for signed types */ - if (ll > data.numeric.upper_bound || - ll < data.numeric.lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %lld and %lld inclusive", - data.numeric.lower_bound, - data.numeric.upper_bound); - *err = loadbuf; - return 0; - } - } + if (!numericBoundaryCheck(data, ll, err)) + return 0; if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) return 0; @@ -1870,33 +1875,8 @@ static int numericConfigSet(typeData data, sds value, char **err) { if (!string2ll(value, sdslen(value),&ll)) return 0; } - if (data.numeric.numeric_type == NUMERIC_TYPE_ULONG_LONG || - data.numeric.numeric_type == NUMERIC_TYPE_UINT || - data.numeric.numeric_type == NUMERIC_TYPE_SIZE_T) { - /* Boundary check for unsigned types */ - unsigned long long ull = ll; - unsigned long long upper_bound = data.numeric.upper_bound; - unsigned long long lower_bound = data.numeric.lower_bound; - if (ull > upper_bound || ull < lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %llu and %llu inclusive", - lower_bound, - upper_bound); - *err = loadbuf; - return 0; - } - } else { - /* Boundary check for signed types */ - if (ll > data.numeric.upper_bound || - ll < data.numeric.lower_bound) { - snprintf(loadbuf, LOADBUF_SIZE, - "argument must be between %lld and %lld inclusive", - data.numeric.lower_bound, - data.numeric.upper_bound); - *err = loadbuf; - return 0; - } - } + if (!numericBoundaryCheck(data, ll, err)) + return 0; if (data.numeric.is_valid_fn && !data.numeric.is_valid_fn(ll, err)) return 0; @@ -2155,32 +2135,32 @@ static int updateTlsCfgBool(int val, int prev, char **err) { standardConfig configs[] = { /* Bool configs */ createBoolConfig("rdbchecksum", NULL, IMMUTABLE_CONFIG, server.rdb_checksum, 1, NULL, NULL), - createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0 ,NULL, NULL), - createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0 ,NULL, NULL), /* Read + parse from threads? */ - createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1 ,NULL, NULL), - createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0 ,NULL, NULL), - createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1 ,NULL, NULL), - createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1 ,NULL, NULL), - createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1 ,NULL, NULL), - createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1 ,NULL, NULL), - createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1 ,NULL, NULL), /* Adapt hz to # of clients.*/ - createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0 ,NULL, NULL), - createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0 ,NULL, NULL), - createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0 ,NULL, NULL), - createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0 ,NULL, NULL), - createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0 ,NULL, NULL), - createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0 ,NULL, NULL), - createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1 ,NULL, NULL), - createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0 ,NULL, NULL), - createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1 ,NULL, NULL), - createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, 1 ,NULL, NULL), - createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1 ,NULL, NULL), - createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1 ,NULL, NULL), - createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, 0 ,NULL, NULL), /* Failover by default. */ - createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, 0 ,NULL, NULL), - createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1 ,NULL, NULL), - createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, 1 ,NULL, NULL), - createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1 ,NULL, NULL), + createBoolConfig("daemonize", NULL, IMMUTABLE_CONFIG, server.daemonize, 0, NULL, NULL), + createBoolConfig("io-threads-do-reads", NULL, IMMUTABLE_CONFIG, server.io_threads_do_reads, 0,NULL, NULL), /* Read + parse from threads? */ + createBoolConfig("lua-replicate-commands", NULL, IMMUTABLE_CONFIG, server.lua_always_replicate_commands, 1, NULL, NULL), + createBoolConfig("always-show-logo", NULL, IMMUTABLE_CONFIG, server.always_show_logo, 0, NULL, NULL), + createBoolConfig("protected-mode", NULL, MODIFIABLE_CONFIG, server.protected_mode, 1, NULL, NULL), + createBoolConfig("rdbcompression", NULL, MODIFIABLE_CONFIG, server.rdb_compression, 1, NULL, NULL), + createBoolConfig("activerehashing", NULL, MODIFIABLE_CONFIG, server.activerehashing, 1, NULL, NULL), + createBoolConfig("stop-writes-on-bgsave-error", NULL, MODIFIABLE_CONFIG, server.stop_writes_on_bgsave_err, 1, NULL, NULL), + createBoolConfig("dynamic-hz", NULL, MODIFIABLE_CONFIG, server.dynamic_hz, 1, NULL, NULL), /* Adapt hz to # of clients.*/ + createBoolConfig("lazyfree-lazy-eviction", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_eviction, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-expire", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_expire, 0, NULL, NULL), + createBoolConfig("lazyfree-lazy-server-del", NULL, MODIFIABLE_CONFIG, server.lazyfree_lazy_server_del, 0, NULL, NULL), + createBoolConfig("repl-disable-tcp-nodelay", NULL, MODIFIABLE_CONFIG, server.repl_disable_tcp_nodelay, 0, NULL, NULL), + createBoolConfig("repl-diskless-sync", NULL, MODIFIABLE_CONFIG, server.repl_diskless_sync, 0, NULL, NULL), + createBoolConfig("gopher-enabled", NULL, MODIFIABLE_CONFIG, server.gopher_enabled, 0, NULL, NULL), + createBoolConfig("aof-rewrite-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.aof_rewrite_incremental_fsync, 1, NULL, NULL), + createBoolConfig("no-appendfsync-on-rewrite", NULL, MODIFIABLE_CONFIG, server.aof_no_fsync_on_rewrite, 0, NULL, NULL), + createBoolConfig("cluster-require-full-coverage", NULL, MODIFIABLE_CONFIG, server.cluster_require_full_coverage, 1, NULL, NULL), + createBoolConfig("rdb-save-incremental-fsync", NULL, MODIFIABLE_CONFIG, server.rdb_save_incremental_fsync, 1, NULL, NULL), + createBoolConfig("aof-load-truncated", NULL, MODIFIABLE_CONFIG, server.aof_load_truncated, 1, NULL, NULL), + createBoolConfig("aof-use-rdb-preamble", NULL, MODIFIABLE_CONFIG, server.aof_use_rdb_preamble, 1, NULL, NULL), + createBoolConfig("cluster-replica-no-failover", "cluster-slave-no-failover", MODIFIABLE_CONFIG, server.cluster_slave_no_failover, 0, NULL, NULL), /* Failover by default. */ + createBoolConfig("replica-lazy-flush", "slave-lazy-flush", MODIFIABLE_CONFIG, server.repl_slave_lazy_flush, 0, NULL, NULL), + createBoolConfig("replica-serve-stale-data", "slave-serve-stale-data", MODIFIABLE_CONFIG, server.repl_serve_stale_data, 1, NULL, NULL), + createBoolConfig("replica-read-only", "slave-read-only", MODIFIABLE_CONFIG, server.repl_slave_ro, 1, NULL, NULL), + createBoolConfig("replica-ignore-maxmemory", "slave-ignore-maxmemory", MODIFIABLE_CONFIG, server.repl_slave_ignore_maxmemory, 1, NULL, NULL), createBoolConfig("jemalloc-bg-thread", NULL, MODIFIABLE_CONFIG, server.jemalloc_bg_thread, 1, NULL, updateJemallocBgThread), createBoolConfig("activedefrag", NULL, MODIFIABLE_CONFIG, server.active_defrag_enabled, 0, isValidActiveDefrag, NULL), createBoolConfig("syslog-enabled", NULL, IMMUTABLE_CONFIG, server.syslog_enabled, 0, NULL, NULL), @@ -2188,94 +2168,94 @@ standardConfig configs[] = { createBoolConfig("appendonly", NULL, MODIFIABLE_CONFIG, server.aof_enabled, 0, NULL, updateAppendonly), /* String Configs */ - createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "" ,NULL, NULL), - createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL ,NULL, NULL), - createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL ,NULL, NULL), - createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL ,NULL, NULL), - createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL ,NULL, NULL), - createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL ,NULL, NULL), - createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL ,NULL, NULL), + createStringConfig("aclfile", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.acl_filename, "", NULL, NULL), + createStringConfig("unixsocket", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.unixsocket, NULL, NULL, NULL), + createStringConfig("pidfile", NULL, IMMUTABLE_CONFIG, EMPTY_STRING_IS_NULL, server.pidfile, NULL, NULL, NULL), + createStringConfig("replica-announce-ip", "slave-announce-ip", MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.slave_announce_ip, NULL, NULL, NULL), + createStringConfig("masteruser", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masteruser, NULL, NULL, NULL), + createStringConfig("masterauth", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.masterauth, NULL, NULL, NULL), + createStringConfig("cluster-announce-ip", NULL, MODIFIABLE_CONFIG, EMPTY_STRING_IS_NULL, server.cluster_announce_ip, NULL, NULL, NULL), createStringConfig("syslog-ident", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.syslog_ident, "redis", NULL, NULL), createStringConfig("dbfilename", NULL, MODIFIABLE_CONFIG, ALLOW_EMPTY_STRING, server.rdb_filename, "dump.rdb", isValidDBfilename, NULL), createStringConfig("appendfilename", NULL, IMMUTABLE_CONFIG, ALLOW_EMPTY_STRING, server.aof_filename, "appendonly.aof", isValidAOFfilename, NULL), /* Enum Configs */ - createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE ,NULL, NULL), - createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0 ,NULL, NULL), - createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED ,NULL, NULL), - createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE ,NULL, NULL), - createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION ,NULL, NULL), - createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC ,NULL, NULL), + createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, server.supervised_mode, SUPERVISED_NONE, NULL, NULL), + createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, server.syslog_facility, LOG_LOCAL0, NULL, NULL), + createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, server.repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL), + createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, server.verbosity, LL_NOTICE, NULL, NULL), + createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, server.maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL), + createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, server.aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL), /* Integer configs */ - createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG ,NULL, NULL), /* TCP port. */ - createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, 1, INTEGER_CONFIG ,NULL, NULL), /* Single threaded by default */ - createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG ,NULL, NULL), /* Slave max data age factor. */ - createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG ,NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ - createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, 25, INTEGER_CONFIG ,NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ - createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG ,NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ - createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG ,NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ - createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, 10, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, 1, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG ,NULL, NULL), /* Default client timeout: infinite */ - createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG ,NULL, NULL), /* TCP listen backlog. */ - createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG ,NULL, NULL), /* Default: Use +10000 offset. */ - createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG ,NULL, NULL), /* Use server.port */ - createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG ,NULL, NULL), - createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG ,NULL, NULL), /* Default: 10% tracking table max fill. */ - createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG ,NULL, NULL), /* From 1 to 10. */ - createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG ,NULL, updateHZ), - createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG ,NULL, updateGoodSlaves), - createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG ,NULL, updateGoodSlaves), + createIntConfig("databases", NULL, IMMUTABLE_CONFIG, 1, INT_MAX, server.dbnum, 16, INTEGER_CONFIG, NULL, NULL), + createIntConfig("port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.port, 6379, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ + createIntConfig("io-threads", NULL, IMMUTABLE_CONFIG, 1, 512, server.io_threads_num, 1, INTEGER_CONFIG, NULL, NULL), /* Single threaded by default */ + createIntConfig("auto-aof-rewrite-percentage", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.aof_rewrite_perc, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-replica-validity-factor", "cluster-slave-validity-factor", MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_slave_validity_factor, 10, INTEGER_CONFIG, NULL, NULL), /* Slave max data age factor. */ + createIntConfig("list-max-ziplist-size", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.list_max_ziplist_size, -2, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-keepalive", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcpkeepalive, 300, INTEGER_CONFIG, NULL, NULL), + createIntConfig("cluster-migration-barrier", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.cluster_migration_barrier, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("active-defrag-cycle-min", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_min, 1, INTEGER_CONFIG, NULL, NULL), /* Default: 1% CPU min (at lower threshold) */ + createIntConfig("active-defrag-cycle-max", NULL, MODIFIABLE_CONFIG, 1, 99, server.active_defrag_cycle_max, 25, INTEGER_CONFIG, NULL, NULL), /* Default: 25% CPU max (at upper threshold) */ + createIntConfig("active-defrag-threshold-lower", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_lower, 10, INTEGER_CONFIG, NULL, NULL), /* Default: don't defrag when fragmentation is below 10% */ + createIntConfig("active-defrag-threshold-upper", NULL, MODIFIABLE_CONFIG, 0, 1000, server.active_defrag_threshold_upper, 100, INTEGER_CONFIG, NULL, NULL), /* Default: maximum defrag force at 100% fragmentation */ + createIntConfig("lfu-log-factor", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_log_factor, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("lfu-decay-time", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.lfu_decay_time, 1, INTEGER_CONFIG, NULL, NULL), + createIntConfig("replica-priority", "slave-priority", MODIFIABLE_CONFIG, 0, INT_MAX, server.slave_priority, 100, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-diskless-sync-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_diskless_sync_delay, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("maxmemory-samples", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.maxmemory_samples, 5, INTEGER_CONFIG, NULL, NULL), + createIntConfig("timeout", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.maxidletime, 0, INTEGER_CONFIG, NULL, NULL), /* Default client timeout: infinite */ + createIntConfig("replica-announce-port", "slave-announce-port", MODIFIABLE_CONFIG, 0, 65535, server.slave_announce_port, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tcp-backlog", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.tcp_backlog, 511, INTEGER_CONFIG, NULL, NULL), /* TCP listen backlog. */ + createIntConfig("cluster-announce-bus-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_bus_port, 0, INTEGER_CONFIG, NULL, NULL), /* Default: Use +10000 offset. */ + createIntConfig("cluster-announce-port", NULL, MODIFIABLE_CONFIG, 0, 65535, server.cluster_announce_port, 0, INTEGER_CONFIG, NULL, NULL), /* Use server.port */ + createIntConfig("repl-timeout", NULL, MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_timeout, 60, INTEGER_CONFIG, NULL, NULL), + createIntConfig("repl-ping-replica-period", "repl-ping-slave-period", MODIFIABLE_CONFIG, 1, INT_MAX, server.repl_ping_slave_period, 10, INTEGER_CONFIG, NULL, NULL), + createIntConfig("list-compress-depth", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.list_compress_depth, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("rdb-key-save-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.rdb_key_save_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("key-load-delay", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.key_load_delay, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("tracking-table-max-fill", NULL, MODIFIABLE_CONFIG, 0, 100, server.tracking_table_max_fill, 10, INTEGER_CONFIG, NULL, NULL), /* Default: 10% tracking table max fill. */ + createIntConfig("active-expire-effort", NULL, MODIFIABLE_CONFIG, 1, 10, server.active_expire_effort, 1, INTEGER_CONFIG, NULL, NULL), /* From 1 to 10. */ + createIntConfig("hz", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.config_hz, CONFIG_DEFAULT_HZ, INTEGER_CONFIG, NULL, updateHZ), + createIntConfig("min-replicas-to-write", "min-slaves-to-write", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_to_write, 0, INTEGER_CONFIG, NULL, updateGoodSlaves), + createIntConfig("min-replicas-max-lag", "min-slaves-max-lag", MODIFIABLE_CONFIG, 0, INT_MAX, server.repl_min_slaves_max_lag, 10, INTEGER_CONFIG, NULL, updateGoodSlaves), /* Unsigned int configs */ createUIntConfig("maxclients", NULL, MODIFIABLE_CONFIG, 1, UINT_MAX, server.maxclients, 10000, INTEGER_CONFIG, NULL, updateMaxclients), /* Unsigned Long configs */ - createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG ,NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ - createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG ,NULL, NULL), + createULongConfig("active-defrag-max-scan-fields", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_max_scan_fields, 1000, INTEGER_CONFIG, NULL, NULL), /* Default: keys with more than 1000 fields will be processed separately */ + createULongConfig("slowlog-max-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.slowlog_max_len, 128, INTEGER_CONFIG, NULL, NULL), /* Long Long configs */ - createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG ,NULL, NULL),/* milliseconds */ - createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG ,NULL, NULL), - createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG ,NULL, NULL), /* Bulk request max size */ - createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG ,NULL, NULL), + createLongLongConfig("lua-time-limit", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.lua_time_limit, 5000, INTEGER_CONFIG, NULL, NULL),/* milliseconds */ + createLongLongConfig("cluster-node-timeout", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.cluster_node_timeout, 15000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("slowlog-log-slower-than", NULL, MODIFIABLE_CONFIG, -1, LLONG_MAX, server.slowlog_log_slower_than, 10000, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL), + createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */ + createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL), createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */ /* Unsigned Long Long configs */ - createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG ,NULL, updateMaxmemory), + createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, ULLONG_MAX, server.maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory), /* Size_t configs */ - createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG ,NULL, NULL), - createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG ,NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ - createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG ,NULL, NULL), - createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG ,NULL, NULL), + createSizeTConfig("hash-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("set-max-intset-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.set_max_intset_entries, 512, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-entries", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_entries, 128, INTEGER_CONFIG, NULL, NULL), + createSizeTConfig("active-defrag-ignore-bytes", NULL, MODIFIABLE_CONFIG, 1, LONG_MAX, server.active_defrag_ignore_bytes, 100<<20, MEMORY_CONFIG, NULL, NULL), /* Default: don't defrag if frag overhead is below 100mb */ + createSizeTConfig("hash-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hash_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("stream-node-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.stream_node_max_bytes, 4096, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("zset-max-ziplist-value", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.zset_max_ziplist_value, 64, MEMORY_CONFIG, NULL, NULL), + createSizeTConfig("hll-sparse-max-bytes", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.hll_sparse_max_bytes, 3000, MEMORY_CONFIG, NULL, NULL), /* Other configs */ - createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60 ,INTEGER_CONFIG ,NULL, NULL), /* Default: 1 hour */ - createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG ,NULL, NULL), + createTimeTConfig("repl-backlog-ttl", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX, server.repl_backlog_time_limit, 60*60, INTEGER_CONFIG, NULL, NULL), /* Default: 1 hour */ + createOffTConfig("auto-aof-rewrite-min-size", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, server.aof_rewrite_min_size, 64*1024*1024, MEMORY_CONFIG, NULL, NULL), #ifdef USE_OPENSSL - createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG ,NULL, NULL), /* TCP port. */ + createIntConfig("tls-port", NULL, IMMUTABLE_CONFIG, 0, 65535, server.tls_port, 0, INTEGER_CONFIG, NULL, NULL), /* TCP port. */ createBoolConfig("tls-cluster", NULL, MODIFIABLE_CONFIG, server.tls_cluster, 0, NULL, NULL), createBoolConfig("tls-replication", NULL, MODIFIABLE_CONFIG, server.tls_replication, 0, NULL, NULL), createBoolConfig("tls-auth-clients", NULL, MODIFIABLE_CONFIG, server.tls_auth_clients, 1, NULL, NULL), From 8e19f9e0481a3b6c42e6543f6a85fb2cc55de94c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 1 Dec 2019 14:57:18 +0200 Subject: [PATCH 308/320] revert an accidental test code change done as part of the tls project it seems that commit 10ffeb03e4908b46c34f72d6b526ffa0834fd3f8 accidentially changed gen_write_load to not use deferred client, which causes them to be slower and not generate high load which they should, making some tests less effecitive --- tests/helpers/gen_write_load.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index fd6aad40c..cbf6651bd 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -4,7 +4,7 @@ set ::tlsdir "tests/tls" proc gen_write_load {host port seconds tls} { set start_time [clock seconds] - set r [redis $host $port 0 $tls] + set r [redis $host $port 1 $tls] $r select 9 while 1 { $r set [expr rand()] [expr rand()] From f6fa999eb49f557803b7ee48a2059c7a32171e4a Mon Sep 17 00:00:00 2001 From: Andy Guan Date: Mon, 2 Dec 2019 00:28:41 -0500 Subject: [PATCH 309/320] Added the missed macro definition in slowlog.h --- src/slowlog.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/slowlog.h b/src/slowlog.h index 655fb25f4..50e543891 100644 --- a/src/slowlog.h +++ b/src/slowlog.h @@ -27,6 +27,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ +#ifndef __SLOWLOG_H__ +#define __SLOWLOG_H__ + #define SLOWLOG_ENTRY_MAX_ARGC 32 #define SLOWLOG_ENTRY_MAX_STRING 128 @@ -47,3 +50,5 @@ void slowlogPushEntryIfNeeded(client *c, robj **argv, int argc, long long durati /* Exported commands */ void slowlogCommand(client *c); + +#endif /* __SLOWLOG_H__ */ From 319b0418fdfeaa2f3197800220cf7ad56dc3639d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Dec 2019 09:13:23 +0100 Subject: [PATCH 310/320] Avoid collision with MacOS LIST_HEAD macro after #6384. --- src/zmalloc.c | 8 ++++++++ src/zmalloc.h | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index ec7fcc030..639a5fe2b 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -388,6 +388,14 @@ int jemalloc_purge() { #endif +#if defined(__APPLE__) +/* For proc_pidinfo() used later in zmalloc_get_smap_bytes_by_field(). + * Note that this file cannot be included in zmalloc.h because it includes + * a Darwin queue.h file where there is a "LIST_HEAD" macro (!) defined + * conficting with Redis user code. */ +#include +#endif + /* Get the sum of the specified field (converted form kb to bytes) in * /proc/self/smaps. The field must be specified with trailing ":" as it * apperas in the smaps output. diff --git a/src/zmalloc.h b/src/zmalloc.h index d32698e62..b136a910d 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -57,7 +57,6 @@ #elif defined(__APPLE__) #include -#include #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) malloc_size(p) #endif From 7450df615080325ca337ca065965fbd86619e1a7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 2 Dec 2019 19:17:35 +0200 Subject: [PATCH 311/320] fix crash in module short read test --- tests/modules/testrdb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/modules/testrdb.c b/tests/modules/testrdb.c index 7c04bb4ef..3150b88e4 100644 --- a/tests/modules/testrdb.c +++ b/tests/modules/testrdb.c @@ -20,7 +20,8 @@ void *testrdb_type_load(RedisModuleIO *rdb, int encver) { long double ld = RedisModule_LoadLongDouble(rdb); if (RedisModule_IsIOError(rdb)) { RedisModuleCtx *ctx = RedisModule_GetContextFromIO(rdb); - RedisModule_FreeString(ctx, str); + if (str) + RedisModule_FreeString(ctx, str); return NULL; } /* Using the values only after checking for io errors. */ From c57ba6d6b48215347fdd1dbc038f83755514cede Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Dec 2019 16:21:23 +0100 Subject: [PATCH 312/320] Modules: create timers in contexts without a client. --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index be64af368..65c0f5b3f 100644 --- a/src/module.c +++ b/src/module.c @@ -5150,7 +5150,7 @@ RedisModuleTimerID RM_CreateTimer(RedisModuleCtx *ctx, mstime_t period, RedisMod timer->module = ctx->module; timer->callback = callback; timer->data = data; - timer->dbid = ctx->client->db->id; + timer->dbid = ctx->client ? ctx->client->db->id : 0; uint64_t expiretime = ustime()+period*1000; uint64_t key; From a64168dc89f9ea660d3f4050e610dfec30ec735b Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Dec 2019 17:52:04 +0100 Subject: [PATCH 313/320] create-cluster script: allow additional options. --- utils/create-cluster/create-cluster | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index 9ffd462ae..ab287403a 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -7,6 +7,7 @@ TIMEOUT=2000 NODES=6 REPLICAS=1 PROTECTED_MODE=yes +ADDITIONAL_OPTIONS="" # You may want to put the above config parameters into config.sh in order to # override the defaults without modifying this script. @@ -24,7 +25,7 @@ then while [ $((PORT < ENDPORT)) != "0" ]; do PORT=$((PORT+1)) echo "Starting $PORT" - ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes + ../../src/redis-server --port $PORT --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS} done exit 0 fi From d4c5516650e616003f123783332bbea8f159fa6b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Dec 2019 10:38:18 +0100 Subject: [PATCH 314/320] Modules: clarify when the disconnection callback is called. --- src/module.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 65c0f5b3f..111e1c202 100644 --- a/src/module.c +++ b/src/module.c @@ -4192,7 +4192,10 @@ void moduleBlockedClientPipeReadable(aeEventLoop *el, int fd, void *privdata, in void unblockClientFromModule(client *c) { RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; - /* Call the disconnection callback if any. */ + /* Call the disconnection callback if any. Note that + * bc->disconnect_callback is set to NULL if the client gets disconnected + * by the module itself, so the callback will NOT get called if this is + * not an actual disconnection event. */ if (bc->disconnect_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.blocked_privdata = bc->privdata; From 82fbd66d7e36c5da9b8e1b8f6fb3240c8b0b54a6 Mon Sep 17 00:00:00 2001 From: Eran Liberty Date: Thu, 5 Dec 2019 13:37:11 +0000 Subject: [PATCH 315/320] - memcpy(&id,ri.key,ri.key_len); + memcpy(&id,ri.key,sizeof(id)); The memcpy from the key to the id reliease on the fact that this key *should* be 8 bytes long as it was entered as such a few lines up the code. BUT if someone will change the code to the point this is no longer true, current code can trash the stack which makes debugging very hard while this fix will result in some garbage id, or even page fault. Both are preferable to stack mangaling. --- src/tracking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index f7f0fc755..acb97800a 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -164,7 +164,7 @@ void trackingInvalidateSlot(uint64_t slot) { raxSeek(&ri,"^",NULL,0); while(raxNext(&ri)) { uint64_t id; - memcpy(&id,ri.key,ri.key_len); + memcpy(&id,ri.key,sizeof(id)); client *c = lookupClientByID(id); if (c == NULL || !(c->flags & CLIENT_TRACKING)) continue; sendTrackingMessage(c,slot); From 09ec3072cf9f08d9d334dbbc5e49a7feca3bbb8e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 9 Dec 2019 10:03:23 +0200 Subject: [PATCH 316/320] Add ULL suffix to CLIENT_TRACKING flag to prevent sign extension the code in: c->flags &= ~(CLIENT_TRACKING|CLIENT_TRACKING_BROKEN_REDIR); will do sign extension and turn on all the high 31 bits no damage so far since we don't have any yet --- src/server.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.h b/src/server.h index d251fa026..f2c93241c 100644 --- a/src/server.h +++ b/src/server.h @@ -239,7 +239,7 @@ typedef long long ustime_t; /* microsecond time type. */ we return single threaded that the client has already pending commands to be executed. */ -#define CLIENT_TRACKING (1<<31) /* Client enabled keys tracking in order to +#define CLIENT_TRACKING (1ULL<<31) /* Client enabled keys tracking in order to perform client side caching. */ #define CLIENT_TRACKING_BROKEN_REDIR (1ULL<<32) /* Target client is invalid. */ From e65c6772fc7cbb32d919eae07b6fd4c788fbfab4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Dec 2019 12:15:38 +0100 Subject: [PATCH 317/320] Modules: more clarification about disconnection callback. --- src/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 111e1c202..f9e0d8036 100644 --- a/src/module.c +++ b/src/module.c @@ -4194,8 +4194,8 @@ void unblockClientFromModule(client *c) { /* Call the disconnection callback if any. Note that * bc->disconnect_callback is set to NULL if the client gets disconnected - * by the module itself, so the callback will NOT get called if this is - * not an actual disconnection event. */ + * by the module itself or because of a timeout, so the callback will NOT + * get called if this is not an actual disconnection event. */ if (bc->disconnect_callback) { RedisModuleCtx ctx = REDISMODULE_CTX_INIT; ctx.blocked_privdata = bc->privdata; From bfc40083ac9d832af05e77663a6d53ff31879fc0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Dec 2019 10:16:25 +0100 Subject: [PATCH 318/320] create-cluster script tailall option. --- utils/create-cluster/create-cluster | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/utils/create-cluster/create-cluster b/utils/create-cluster/create-cluster index ab287403a..4aef0816a 100755 --- a/utils/create-cluster/create-cluster +++ b/utils/create-cluster/create-cluster @@ -71,6 +71,12 @@ then exit 0 fi +if [ "$1" == "tailall" ] +then + tail -f *.log + exit 0 +fi + if [ "$1" == "call" ] then while [ $((PORT < ENDPORT)) != "0" ]; do @@ -101,5 +107,6 @@ echo "create -- Create a cluster using redis-cli --cluster create." echo "stop -- Stop Redis Cluster instances." echo "watch -- Show CLUSTER NODES output (first 30 lines) of first node." echo "tail -- Run tail -f of instance at base port + ID." +echo "tailall -- Run tail -f for all the log files at once." echo "clean -- Remove all instances data, logs, configs." echo "clean-logs -- Remove just instances logs." From 5b203adbe08371ba8c7831a9eebe4e1dd952ac54 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Dec 2019 10:17:23 +0100 Subject: [PATCH 319/320] Clients connected and bytes used is too spammy for verbose. This message is there for ten years, but is hardly useful. Moreover it is likely that it will fill an entire disk if log ratation is not configured, for no good reasons. --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 72f136446..adb7ce6be 100644 --- a/src/server.c +++ b/src/server.c @@ -1942,7 +1942,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Show information about connected clients */ if (!server.sentinel_mode) { run_with_period(5000) { - serverLog(LL_VERBOSE, + serverLog(LL_DEBUG, "%lu clients connected (%lu replicas), %zu bytes in use", listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), From 4e1326940cc28cc46c0d248cded0f4eaba53ead5 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 11 Dec 2019 12:35:00 +0200 Subject: [PATCH 320/320] fix leak in RM_LoadDataTypeFromString, and save --- src/module.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 111e1c202..dd80524ca 100644 --- a/src/module.c +++ b/src/module.c @@ -3963,6 +3963,7 @@ void RM_DigestEndSequence(RedisModuleDigest *md) { void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType *mt) { rio payload; RedisModuleIO io; + void *ret; rioInitWithBuffer(&payload, str->ptr); moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); @@ -3971,7 +3972,12 @@ void *RM_LoadDataTypeFromString(const RedisModuleString *str, const moduleType * * need to make sure we read the same. */ io.ver = 2; - return mt->rdb_load(&io,0); + ret = mt->rdb_load(&io,0); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } + return ret; } /* Encode a module data type 'mt' value 'data' into serialized form, and return it @@ -3989,11 +3995,15 @@ RedisModuleString *RM_SaveDataTypeToString(RedisModuleCtx *ctx, void *data, cons rioInitWithBuffer(&payload,sdsempty()); moduleInitIOContext(io,(moduleType *)mt,&payload,NULL); mt->rdb_save(&io,data); + if (io.ctx) { + moduleFreeContext(io.ctx); + zfree(io.ctx); + } if (io.error) { return NULL; } else { robj *str = createObject(OBJ_STRING,payload.io.buffer.ptr); - autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str); + if (ctx != NULL) autoMemoryAdd(ctx,REDISMODULE_AM_STRING,str); return str; } }