From 3447062d5e02cf14fd3d7765fa39a65de15f8c15 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 16 Jan 2015 09:03:00 +0000 Subject: [PATCH 001/616] Make some defaults explicit in the sentinel.conf for package maintainers This may look a little pointless (and it is a complete no-op change here) but as package maintainers need to modify these lines to actually daemonize (etc. etc) but it's far preferable if the diff is restricted to actually changing just that bit, not adding docs, etc. The less diff the better, in general. Signed-off-by: Chris Lamb --- sentinel.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index 39d1044e2..d627b8536 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -4,6 +4,31 @@ # The port that this sentinel instance will run on port 26379 +# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when +# daemonized. +daemonize no + +# When running daemonized, Redis Sentinel writes a pid file in +# /var/run/redis-sentinel.pid by default. You can specify a custom pid file +# location here. +pidfile /var/run/redis-sentinel.pid + +# By default Redis Sentinel listens for connections from all the network +# interfaces available on the server. It is possible to listen to just one or +# multiple interfaces using the "bind" configuration directive, followed by one +# or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 + +# Specify the log file name. Also the empty string can be used to force +# Sentinel to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + # sentinel announce-ip # sentinel announce-port # From eaeba1b2c85812c6b16a8ef50bf889d72804db09 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Wed, 4 Feb 2015 18:29:22 +0000 Subject: [PATCH 002/616] Tidy grammar in CONFIG SET maxmemory warning. Signed-off-by: Chris Lamb --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 8255a56b7..f0a019517 100644 --- a/src/config.c +++ b/src/config.c @@ -648,7 +648,7 @@ void configSetCommand(redisClient *c) { server.maxmemory = ll; if (server.maxmemory) { if (server.maxmemory < zmalloc_used_memory()) { - redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); + redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); } freeMemoryIfNeeded(); } From 82ae4f30ed8d2c55c27a465429adee572654bdb2 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 29 Apr 2016 16:45:53 +0100 Subject: [PATCH 003/616] Use SOURCE_DATE_EPOCH over unreproducible uname + date calls. See for more details. Signed-off-by: Chris Lamb --- src/mkreleasehdr.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index 1ae95886b..e6d558b17 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -2,6 +2,9 @@ GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` +if [ -n "$SOURCE_DATE_EPOCH" ]; then + BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u %s) +fi test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ (cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already up-to-date From 4317e2131f4355046fd44141c8b6515b81dc79c0 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sun, 10 Dec 2017 17:54:56 +0200 Subject: [PATCH 004/616] Standardizes `MEMORY HELP` subcommand --- src/object.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/object.c b/src/object.c index 43ab6b5f0..52d2d4662 100644 --- a/src/object.c +++ b/src/object.c @@ -1120,7 +1120,18 @@ NULL void memoryCommand(client *c) { robj *o; - if (!strcasecmp(c->argv[1]->ptr,"usage") && c->argc >= 3) { + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + + const char *help[] = { +"doctor - Return memory problems reports.", +"malloc-stats -- Return internal statistics report from the memory allocator.", +"purge -- Attempt to purge dirty pages for reclamation by the allocator.", +"stats -- Return information about the memory usage of the server.", +"usage [samples ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", +NULL + }; + addReplyHelp(c, help); + } else if (!strcasecmp(c->argv[1]->ptr,"usage") && c->argc >= 3) { long long samples = OBJ_COMPUTE_SIZE_DEF_SAMPLES; for (int j = 3; j < c->argc; j++) { if (!strcasecmp(c->argv[j]->ptr,"samples") && @@ -1234,19 +1245,7 @@ void memoryCommand(client *c) { addReply(c, shared.ok); /* Nothing to do for other allocators. */ #endif - } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) { - addReplyMultiBulkLen(c,5); - addReplyBulkCString(c, -"MEMORY DOCTOR - Outputs memory problems report"); - addReplyBulkCString(c, -"MEMORY USAGE [SAMPLES ] - Estimate memory usage of key"); - addReplyBulkCString(c, -"MEMORY STATS - Show memory usage details"); - addReplyBulkCString(c, -"MEMORY PURGE - Ask the allocator to release memory"); - addReplyBulkCString(c, -"MEMORY MALLOC-STATS - Show allocator internal stats"); } else { - addReplyError(c,"Syntax error. Try MEMORY HELP"); + addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr); } } From 7820377d0084ac400345047ab67c0752e98897d8 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 15 Dec 2017 21:19:41 +0200 Subject: [PATCH 005/616] Uppercases subcommands in MEMORY HELP --- src/object.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/object.c b/src/object.c index 52d2d4662..606cdd7e5 100644 --- a/src/object.c +++ b/src/object.c @@ -1123,11 +1123,11 @@ void memoryCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"doctor - Return memory problems reports.", -"malloc-stats -- Return internal statistics report from the memory allocator.", -"purge -- Attempt to purge dirty pages for reclamation by the allocator.", -"stats -- Return information about the memory usage of the server.", -"usage [samples ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", +"DOCTOR - Return memory problems reports.", +"MALLOC-STATS -- Return internal statistics report from the memory allocator.", +"PURGE -- Attempt to purge dirty pages for reclamation by the allocator.", +"STATS -- Return information about the memory usage of the server.", +"USAGE [SAMPLES ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", NULL }; addReplyHelp(c, help); From d49bfc40808652389e3e3c3a0db3667153c3c14f Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 15 Dec 2017 21:21:12 +0200 Subject: [PATCH 006/616] Uppercases subcommands in OBJECT HELP --- src/object.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index 606cdd7e5..b979b458a 100644 --- a/src/object.c +++ b/src/object.c @@ -1073,10 +1073,10 @@ void objectCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"encoding -- Return the kind of internal representation used in order to store the value associated with a key.", -"freq -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key.", -"idletime -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key.", -"refcount -- Return the number of references of the value associated with the specified key.", +"ENCODING -- Return the kind of internal representation used in order to store the value associated with a key.", +"FREQ -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key.", +"IDLETIME -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key.", +"REFCOUNT -- Return the number of references of the value associated with the specified key.", NULL }; addReplyHelp(c, help); From 290a63dc54f9cd2a61681be3849f1d9d481aa060 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 6 Mar 2018 19:34:44 +0700 Subject: [PATCH 007/616] Don't call sdscmp() with shared.maxstring or shared.minstring --- src/t_zset.c | 14 ++++++-------- tests/unit/type/zset.tcl | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index f7f4c6eb2..50652244b 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -519,12 +519,12 @@ int zslParseLexRangeItem(robj *item, sds *dest, int *ex) { switch(c[0]) { case '+': if (c[1] != '\0') return C_ERR; - *ex = 0; + *ex = 1; *dest = shared.maxstring; return C_OK; case '-': if (c[1] != '\0') return C_ERR; - *ex = 0; + *ex = 1; *dest = shared.minstring; return C_OK; case '(': @@ -597,9 +597,8 @@ int zslIsInLexRange(zskiplist *zsl, zlexrangespec *range) { zskiplistNode *x; /* Test for ranges that will always be empty. */ - if (sdscmplex(range->min,range->max) > 1 || - (sdscmp(range->min,range->max) == 0 && - (range->minex || range->maxex))) + int cmp = sdscmplex(range->min,range->max); + if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex))) return 0; x = zsl->tail; if (x == NULL || !zslLexValueGteMin(x->ele,range)) @@ -872,9 +871,8 @@ int zzlIsInLexRange(unsigned char *zl, zlexrangespec *range) { unsigned char *p; /* Test for ranges that will always be empty. */ - if (sdscmplex(range->min,range->max) > 1 || - (sdscmp(range->min,range->max) == 0 && - (range->minex || range->maxex))) + int cmp = sdscmplex(range->min,range->max); + if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex))) return 0; p = ziplistIndex(zl,-2); /* Last element. */ diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 564825ae9..d8f3cfa53 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -388,7 +388,7 @@ start_server {tags {"zset"}} { 0 omega} } - test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZCOUNT basics" { + test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZLEXCOUNT basics" { create_default_lex_zset # inclusive range @@ -416,6 +416,22 @@ start_server {tags {"zset"}} { assert_equal {} [r zrevrangebylex zset \[elez \[elex] assert_equal {} [r zrevrangebylex zset (hill (omega] } + + test "ZLEXCOUNT advanced" { + create_default_lex_zset + + assert_equal 9 [r zlexcount zset - +] + assert_equal 0 [r zlexcount zset + -] + assert_equal 0 [r zlexcount zset + \[c] + assert_equal 0 [r zlexcount zset \[c -] + assert_equal 8 [r zlexcount zset \[bar +] + assert_equal 5 [r zlexcount zset \[bar \[foo] + assert_equal 4 [r zlexcount zset \[bar (foo] + assert_equal 4 [r zlexcount zset (bar \[foo] + assert_equal 3 [r zlexcount zset (bar (foo] + assert_equal 5 [r zlexcount zset - (foo] + assert_equal 1 [r zlexcount zset (maxstring +] + } test "ZRANGEBYSLEX with LIMIT" { create_default_lex_zset From fbef85ca5aca774c7533c6e0760edfd6258948c7 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 29 Mar 2018 23:20:58 +0800 Subject: [PATCH 008/616] debug: avoid free client unexpectedly when reload & loadaof --- src/debug.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 293dbe36e..5d5029f5a 100644 --- a/src/debug.c +++ b/src/debug.c @@ -340,7 +340,10 @@ NULL return; } emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); - if (rdbLoad(server.rdb_filename,NULL) != C_OK) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + int ret = rdbLoad(server.rdb_filename,NULL); + aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); return; } @@ -349,7 +352,10 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { if (server.aof_state == AOF_ON) flushAppendOnlyFile(1); emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); - if (loadAppendOnlyFile(server.aof_filename) != C_OK) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + int ret = loadAppendOnlyFile(server.aof_filename); + aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + if (ret != C_OK) { addReply(c,shared.err); return; } From d56f4b4122aa51f97f5284e6e449943dd46ad659 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 2 Apr 2018 18:36:17 +0300 Subject: [PATCH 009/616] Add redis-cli support for diskless replication (CAPA EOF) when setting repl-diskless-sync yes, and sending SYNC. redis-cli needs to be able to understand the EOF marker protocol in order to be able to skip or download the rdb file --- src/redis-cli.c | 112 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d80973e75..6fffa9024 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1793,9 +1793,31 @@ static void latencyDistMode(void) { * Slave mode *--------------------------------------------------------------------------- */ +#define RDB_EOF_MARK_SIZE 40 + +void sendReplconf(const char* arg1, const char* arg2) { + printf("sending REPLCONF %s %s\n", arg1, arg2); + redisReply *reply = redisCommand(context, "REPLCONF %s %s", arg1, arg2); + + /* Handle any error conditions */ + if(reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + exit(1); + } else if(reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "REPLCONF %s error: %s\n", arg1, reply->str); + /* non fatal, old versions may not support it */ + } + freeReplyObject(reply); +} + +void sendCapa() { + sendReplconf("capa", "eof"); +} + /* Sends SYNC and reads the number of bytes in the payload. Used both by - * slaveMode() and getRDB(). */ -unsigned long long sendSync(int fd) { + * slaveMode() and getRDB(). + * returns 0 in case an EOF marker is used. */ +unsigned long long sendSync(int fd, char *out_eof) { /* To start we need to send the SYNC command and return the payload. * The hiredis client lib does not understand this part of the protocol * and we don't want to mess with its buffers, so everything is performed @@ -1825,17 +1847,33 @@ unsigned long long sendSync(int fd) { printf("SYNC with master failed: %s\n", buf); exit(1); } + if (strncmp(buf+1,"EOF:",4) == 0 && strlen(buf+5) >= RDB_EOF_MARK_SIZE) { + memcpy(out_eof, buf+5, RDB_EOF_MARK_SIZE); + return 0; + } return strtoull(buf+1,NULL,10); } static void slaveMode(void) { int fd = context->fd; - unsigned long long payload = sendSync(fd); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(fd, eofmark); char buf[1024]; int original_output = config.output; - fprintf(stderr,"SYNC with master, discarding %llu " - "bytes of bulk transfer...\n", payload); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC with master, discarding " + "bytes of bulk transfer until EOF marker...\n"); + } else { + fprintf(stderr,"SYNC with master, discarding %llu " + "bytes of bulk transfer...\n", payload); + } + /* Discard the payload. */ while(payload) { @@ -1847,8 +1885,29 @@ static void slaveMode(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } } - fprintf(stderr,"SYNC done. Logging commands from master.\n"); + + if (usemark) { + unsigned long long offset = ULLONG_MAX - payload; + fprintf(stderr,"SYNC done after %llu bytes. Logging commands from master.\n", offset); + /* put the slave online */ + sleep(1); + sendReplconf("ACK", "0"); + } else + fprintf(stderr,"SYNC done. Logging commands from master.\n"); /* Now we can use hiredis to read the incoming protocol. */ config.output = OUTPUT_CSV; @@ -1865,11 +1924,22 @@ static void slaveMode(void) { static void getRDB(void) { int s = context->fd; int fd; - unsigned long long payload = sendSync(s); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(s, eofmark); char buf[4096]; - fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", - payload, config.rdb_filename); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC sent to master, writing bytes of bulk transfer until EOF marker to '%s'\n", + config.rdb_filename); + } else { + fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", + payload, config.rdb_filename); + } /* Write to file. */ if (!strcmp(config.rdb_filename,"-")) { @@ -1898,11 +1968,31 @@ static void getRDB(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } + } + if (usemark) { + payload = ULLONG_MAX - payload - RDB_EOF_MARK_SIZE; + if (ftruncate(fd, payload) == -1) + fprintf(stderr,"ftruncate failed: %s.\n", strerror(errno)); + fprintf(stderr,"Transfer finished with success after %llu bytes\n", payload); + } else { + fprintf(stderr,"Transfer finished with success.\n"); } close(s); /* Close the file descriptor ASAP as fsync() may take time. */ fsync(fd); close(fd); - fprintf(stderr,"Transfer finished with success.\n"); exit(0); } @@ -2893,12 +2983,14 @@ int main(int argc, char **argv) { /* Slave mode */ if (config.slave_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); slaveMode(); } /* Get RDB mode. */ if (config.getrdb_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); getRDB(); } From 49890c8ee9776f13aaabf7fe76d796a89de6bf1a Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Mon, 30 Apr 2018 19:33:01 +0300 Subject: [PATCH 010/616] Adds memory information about the script's cache to INFO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementation notes: as INFO is "already broken", I didn't want to break it further. Instead of computing the server.lua_script dict size on every call, I'm keeping a running sum of the body's length and dict overheads. This implementation is naive as it **does not** take into consideration dict rehashing, but that inaccuracy pays off in speed ;) Demo time: ```bash $ redis-cli info memory | grep "script" used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "" 0 ; redis-cli info memory | grep "script" (nil) used_memory_scripts:120 used_memory_scripts_human:120B number_of_cached_scripts:1 $ redis-cli script flush ; redis-cli info memory | grep "script" OK used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "return('Hello, Script Cache :)')" 0 ; redis-cli info memory | grep "script" "Hello, Script Cache :)" used_memory_scripts:152 used_memory_scripts_human:152B number_of_cached_scripts:1 $ redis-cli eval "return redis.sha1hex(\"return('Hello, Script Cache :)')\")" 0 ; redis-cli info memory | grep "script" "1be72729d43da5114929c1260a749073732dc822" used_memory_scripts:232 used_memory_scripts_human:232B number_of_cached_scripts:2 ✔ 19:03:54 redis [lua_scripts-in-info-memory L ✚…⚑] $ redis-cli evalsha 1be72729d43da5114929c1260a749073732dc822 0 "Hello, Script Cache :)" ``` --- src/scripting.c | 3 +++ src/server.c | 8 ++++++++ src/server.h | 1 + 3 files changed, 12 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 3c0597c7a..e131d8ae0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,6 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); + server.lua_scripts_mem = sizeof(dict); /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1073,6 +1074,7 @@ void scriptingInit(int setup) { * This function is used in order to reset the scripting environment. */ void scriptingRelease(void) { dictRelease(server.lua_scripts); + server.lua_scripts_mem = 0; lua_close(server.lua); } @@ -1207,6 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); + server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); incrRefCount(body); return sha; } diff --git a/src/server.c b/src/server.c index 7f51778d7..9bab389c0 100644 --- a/src/server.c +++ b/src/server.c @@ -2994,6 +2994,7 @@ sds genRedisInfoString(char *section) { char peak_hmem[64]; char total_system_hmem[64]; char used_memory_lua_hmem[64]; + char used_memory_scripts_hmem[64]; char used_memory_rss_hmem[64]; char maxmemory_hmem[64]; size_t zmalloc_used = zmalloc_used_memory(); @@ -3013,6 +3014,7 @@ sds genRedisInfoString(char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); + bytesToHuman(used_memory_scripts_hmem,server.lua_scripts_mem); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -3037,6 +3039,9 @@ sds genRedisInfoString(char *section) { "total_system_memory_human:%s\r\n" "used_memory_lua:%lld\r\n" "used_memory_lua_human:%s\r\n" + "used_memory_scripts:%lld\r\n" + "used_memory_scripts_human:%s\r\n" + "number_of_cached_scripts:%lu\r\n" "maxmemory:%lld\r\n" "maxmemory_human:%s\r\n" "maxmemory_policy:%s\r\n" @@ -3069,6 +3074,9 @@ sds genRedisInfoString(char *section) { total_system_hmem, memory_lua, used_memory_lua_hmem, + server.lua_scripts_mem, + used_memory_scripts_hmem, + dictSize(server.lua_scripts), server.maxmemory, maxmemory_hmem, evict_policy, diff --git a/src/server.h b/src/server.h index 0e9c3f285..260ba80d7 100644 --- a/src/server.h +++ b/src/server.h @@ -1203,6 +1203,7 @@ struct redisServer { client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ mstime_t lua_time_start; /* Start time of script, milliseconds time */ int lua_write_dirty; /* True if a write command was called during the From d4c5fc57db0c60780cbce8dfe2b6a5e457cedf83 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:49:00 +0200 Subject: [PATCH 011/616] clientsCronTrackExpansiveClients() skeleton and ideas. --- src/server.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/server.c b/src/server.c index 06826a356..53a8e5368 100644 --- a/src/server.c +++ b/src/server.c @@ -885,6 +885,28 @@ int clientsCronResizeQueryBuffer(client *c) { return 0; } +/* This function is used in order to track clients using the biggest amount + * of memory in the latest few seconds. This way we can provide such information + * in the INFO output (clients section), without having to do an O(N) scan for + * all the clients. + * + * This is how it works. We have an array of CLIENTS_PEAK_MEM_USAGE_SLOTS slots + * where we track, for each, the biggest client output and input buffers we + * saw in that slot. Every slot correspond to one of the latest seconds, since + * the array is indexed by doing UNIXTIME % CLIENTS_PEAK_MEM_USAGE_SLOTS. + * + * When we want to know what was recently the peak memory usage, we just scan + * such few slots searching for the maximum value. */ +#define CLIENTS_PEAK_MEM_USAGE_SLOTS 10 +size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; +size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; + +int clientsCronTrackExpansiveClients(client *c) { + size_t in_usage = sdsAllocSize(c->querybuf); + size_t out_usage = getClientOutputBufferMemoryUsage(c); + return 0; /* This function never terminates the client. */ +} + #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { /* Make sure to process at least numclients/server.hz of clients @@ -917,6 +939,7 @@ void clientsCron(void) { * terminated. */ if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; + if (clientsCronTrackExpansiveClients(c)) continue; } } From 85a1b4f807fd7651184363fadb4a8cdac864adc6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:54:15 +0200 Subject: [PATCH 012/616] clientsCronTrackExpansiveClients() actual implementation. --- src/server.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 53a8e5368..b14d0740e 100644 --- a/src/server.c +++ b/src/server.c @@ -897,13 +897,26 @@ int clientsCronResizeQueryBuffer(client *c) { * * When we want to know what was recently the peak memory usage, we just scan * such few slots searching for the maximum value. */ -#define CLIENTS_PEAK_MEM_USAGE_SLOTS 10 +#define CLIENTS_PEAK_MEM_USAGE_SLOTS 8 size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; int clientsCronTrackExpansiveClients(client *c) { size_t in_usage = sdsAllocSize(c->querybuf); size_t out_usage = getClientOutputBufferMemoryUsage(c); + int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS; + int j = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; + + /* Always zero the next sample, so that when we switch to that second, we'll + * only register samples that are greater in that second without considering + * the history of such slot. */ + ClientsPeakMemInput[j] = 0; + ClientsPeakMemOutput[j] = 0; + + /* Track the biggest values observed so far in this slot. */ + if (in_usage > ClientsPeakMemInput[i]) ClientsPeakMemInput[i] = in_usage; + if (out_usage > ClientsPeakMemOutput[i]) ClientsPeakMemOutput[i] = out_usage; + return 0; /* This function never terminates the client. */ } From 8d617596f1ba5eab71820b0e8faafad893a40176 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:58:04 +0200 Subject: [PATCH 013/616] Implement a function to retrieve the expansive clients mem usage. --- src/server.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/server.c b/src/server.c index b14d0740e..15417a757 100644 --- a/src/server.c +++ b/src/server.c @@ -920,6 +920,18 @@ int clientsCronTrackExpansiveClients(client *c) { return 0; /* This function never terminates the client. */ } +/* Return the max samples in the memory usage of clients tracked by + * the function clientsCronTrackExpansiveClients(). */ +void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { + size_t i = 0, o = 0; + for (int j = 0; j < CLIENTS_PEAK_MEM_USAGE_SLOTS; j++) { + if (ClientsPeakMemInput[j] > i) i = ClientsPeakMemInput[j]; + if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[o]; + } + *in_usage = i; + *out_usage = o; +} + #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { /* Make sure to process at least numclients/server.hz of clients From 8f7e496ba50279fbc384cc391ebb45f7b4aaa8e8 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:59:13 +0200 Subject: [PATCH 014/616] Rename var in clientsCronTrackExpansiveClients() for clarity. --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 15417a757..3ece41b7e 100644 --- a/src/server.c +++ b/src/server.c @@ -905,13 +905,13 @@ int clientsCronTrackExpansiveClients(client *c) { size_t in_usage = sdsAllocSize(c->querybuf); size_t out_usage = getClientOutputBufferMemoryUsage(c); int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS; - int j = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; + int zeroidx = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; /* Always zero the next sample, so that when we switch to that second, we'll * only register samples that are greater in that second without considering * the history of such slot. */ - ClientsPeakMemInput[j] = 0; - ClientsPeakMemOutput[j] = 0; + ClientsPeakMemInput[zeroidx] = 0; + ClientsPeakMemOutput[zeroidx] = 0; /* Track the biggest values observed so far in this slot. */ if (in_usage > ClientsPeakMemInput[i]) ClientsPeakMemInput[i] = in_usage; From de5ca516aed4065e7b7e05d692996cfc028b0f32 Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Thu, 19 Jul 2018 20:48:08 +0800 Subject: [PATCH 015/616] Streams: ID of xclaim command starts from the sixth argument. --- src/t_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 201509c7e..abef73c43 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1988,7 +1988,7 @@ void xclaimCommand(client *c) { * the client successfully claimed some message, so it should be * executed in a "all or nothing" fashion. */ int j; - for (j = 4; j < c->argc; j++) { + for (j = 5; j < c->argc; j++) { streamID id; if (streamParseIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; } From ea3a20c5d05869a498d037f67f92fd05287f9428 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:16:19 +0200 Subject: [PATCH 016/616] Change INFO CLIENTS sections to report pre-computed max/min client buffers. --- src/server.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index 3ece41b7e..8b1cabecf 100644 --- a/src/server.c +++ b/src/server.c @@ -3081,17 +3081,17 @@ sds genRedisInfoString(char *section) { /* Clients */ if (allsections || defsections || !strcasecmp(section,"clients")) { - unsigned long lol, bib; - getClientsMaxBuffers(&lol,&bib); + size_t maxin, maxout; + getExpansiveClientsInfo(&maxin,&maxout); if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Clients\r\n" "connected_clients:%lu\r\n" - "client_longest_output_list:%lu\r\n" - "client_biggest_input_buf:%lu\r\n" + "client_max_input_buffer:%zu\r\n" + "client_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n", listLength(server.clients)-listLength(server.slaves), - lol, bib, + maxin, maxout, server.blocked_clients); } From 0cf3794e6e4807caaf1c78a62dd7843174c08f70 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:34:15 +0200 Subject: [PATCH 017/616] Fix wrong array index variable in getExpansiveClientsInfo(). --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 8b1cabecf..36f72f941 100644 --- a/src/server.c +++ b/src/server.c @@ -926,7 +926,7 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { size_t i = 0, o = 0; for (int j = 0; j < CLIENTS_PEAK_MEM_USAGE_SLOTS; j++) { if (ClientsPeakMemInput[j] > i) i = ClientsPeakMemInput[j]; - if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[o]; + if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[j]; } *in_usage = i; *out_usage = o; From be88c0b16a53f5763d8fc1ae683f99ee39b0d68e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:38:17 +0200 Subject: [PATCH 018/616] Rename INFO CLIENT max buffers field names for correctness. They are actually delayed a few seconds, so let's call them "recent". --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 36f72f941..8d49b5bf0 100644 --- a/src/server.c +++ b/src/server.c @@ -3087,8 +3087,8 @@ sds genRedisInfoString(char *section) { info = sdscatprintf(info, "# Clients\r\n" "connected_clients:%lu\r\n" - "client_max_input_buffer:%zu\r\n" - "client_max_output_buffer:%zu\r\n" + "client_recent_max_input_buffer:%zu\r\n" + "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, From aba6855282759feed747fb80cd947f86d5039335 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Jul 2018 09:36:44 +0200 Subject: [PATCH 019/616] Clarify that clientsCronTrackExpansiveClients() indexes may jump ahead. --- src/server.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 8d49b5bf0..1cb7a9e8e 100644 --- a/src/server.c +++ b/src/server.c @@ -909,7 +909,15 @@ int clientsCronTrackExpansiveClients(client *c) { /* Always zero the next sample, so that when we switch to that second, we'll * only register samples that are greater in that second without considering - * the history of such slot. */ + * the history of such slot. + * + * Note: our index may jump to any random position if serverCron() is not + * called for some reason with the normal frequency, for instance because + * some slow command is called taking multiple seconds to execute. In that + * case our array may end containing data which is potentially older + * than CLIENTS_PEAK_MEM_USAGE_SLOTS seconds: however this is not a problem + * since here we want just to track if "recently" there were very expansive + * clients from the POV of memory usage. */ ClientsPeakMemInput[zeroidx] = 0; ClientsPeakMemOutput[zeroidx] = 0; From 4ff47a0b9bf4d032891e1956c74a5556e443b7a0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Jul 2018 09:46:18 +0200 Subject: [PATCH 020/616] Top comment clientsCron(). --- src/server.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index 1cb7a9e8e..93c0a46fd 100644 --- a/src/server.c +++ b/src/server.c @@ -940,12 +940,27 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { *out_usage = o; } +/* This function is called by serverCron() and is used in order to perform + * operations on clients that are important to perform constantly. For instance + * we use this function in order to disconnect clients after a timeout, including + * clients blocked in some blocking command with a non-zero timeout. + * + * The function makes some effort to process all the clients every second, even + * if this cannot be strictly guaranteed, since serverCron() may be called with + * an actual frequency lower than server.hz in case of latency events like slow + * commands. + * + * It is very important for this function, and the functions it calls, to be + * very fast: sometimes Redis has tens of hundreds of connected clients, and the + * default server.hz value is 10, so sometimes here we need to process thousands + * of clients per second, turning this function into a source of latency. + */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { - /* Make sure to process at least numclients/server.hz of clients - * per call. Since this function is called server.hz times per second - * we are sure that in the worst case we process all the clients in 1 - * second. */ + /* Try to process at least numclients/server.hz of clients + * per call. Since normally (if there are no big latency events) this + * function is called server.hz times per second, in the average case we + * process all the clients in 1 second. */ int numclients = listLength(server.clients); int iterations = numclients/server.hz; mstime_t now = mstime(); From dd760bd5e6bc742932d4177d7cd691c0f195aa70 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Sat, 21 Jul 2018 08:48:51 +0800 Subject: [PATCH 021/616] Consider aof write error as well as rdb in lua script. --- src/scripting.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 328e3d681..23ff96c53 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -493,11 +493,21 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { { luaPushError(lua, shared.roslaveerr->ptr); goto cleanup; - } else if (server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) + } else if ((server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 && + server.lastbgsave_status == C_ERR) || + (server.aof_state != AOF_OFF && + server.aof_last_write_status == C_ERR)) { - luaPushError(lua, shared.bgsaveerr->ptr); + if (server.aof_last_write_status == C_OK) { + luaPushError(lua, shared.bgsaveerr->ptr); + } else { + sds aof_write_err = sdscatfmt(sdsempty(), + "-MISCONF Errors writing to the AOF file: %s\r\n", + strerror(server.aof_last_write_errno)); + luaPushError(lua, aof_write_err); + sdsfree(aof_write_err); + } goto cleanup; } } From 4017a11144cddfc702afd038fec1461eb6e68811 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Sat, 21 Jul 2018 10:00:32 +0800 Subject: [PATCH 022/616] Do not migrate already expired keys. --- src/cluster.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index e568f68a6..5085c2a6a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5146,6 +5146,8 @@ try_again: serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); } + int num_keys_non_expired = 0; + /* Create RESTORE payload and generate the protocol to call the command. */ for (j = 0; j < num_keys; j++) { long long ttl = 0; @@ -5153,8 +5155,10 @@ try_again: if (expireat != -1) { ttl = expireat-mstime(); + if (ttl < 0) continue; if (ttl < 1) ttl = 1; } + num_keys_non_expired++; serverAssertWithInfo(c,NULL, rioWriteBulkCount(&cmd,'*',replace ? 5 : 4)); @@ -5217,9 +5221,9 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ - if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); + if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys_non_expired+1)); - for (j = 0; j < num_keys; j++) { + for (j = 0; j < num_keys_non_expired; j++) { if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; From bb5b8b3a6ff30d2b324a732bf7f4519443b354ac Mon Sep 17 00:00:00 2001 From: dsomeshwar Date: Sat, 21 Jul 2018 23:42:08 +0530 Subject: [PATCH 023/616] removing redundant check --- src/util.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/util.c b/src/util.c index ca5435026..dd776162e 100644 --- a/src/util.c +++ b/src/util.c @@ -368,9 +368,6 @@ int string2ll(const char *s, size_t slen, long long *value) { if (p[0] >= '1' && p[0] <= '9') { v = p[0]-'0'; p++; plen++; - } else if (p[0] == '0' && slen == 1) { - *value = 0; - return 1; } else { return 0; } From 780815dd6ed62a0744bdbea90c239c5dcf792e60 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 22 Jul 2018 10:17:35 +0300 Subject: [PATCH 024/616] fix recursion typo in zmalloc_usable --- src/zmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 0d9917510..308774d86 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -183,7 +183,7 @@ size_t zmalloc_size(void *ptr) { return size+PREFIX_SIZE; } size_t zmalloc_usable(void *ptr) { - return zmalloc_usable(ptr)-PREFIX_SIZE; + return zmalloc_size(ptr)-PREFIX_SIZE; } #endif From 9674ede85ad5c2e6dcc6d47e78ffc60c92b90504 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 22 Jul 2018 17:52:09 +0800 Subject: [PATCH 025/616] Streams: skip master fileds only when we are going forward in streamIteratorGetID --- src/t_stream.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 201509c7e..15a41bedf 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -548,13 +548,16 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { si->master_fields_count = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ si->master_fields_start = si->lp_ele; - /* Skip master fileds to seek the first entry. */ - for (uint64_t i = 0; i < si->master_fields_count; i++) - si->lp_ele = lpNext(si->lp,si->lp_ele); - /* We are now pointing the zero term of the master entry. If + /* We are now pointing the start filed of the master entry. If * we are iterating in reverse order, we need to seek the * end of the listpack. */ - if (si->rev) si->lp_ele = lpLast(si->lp); + if (si->rev) { + si->lp_ele = lpLast(si->lp); + } else { + /* Skip master fileds to seek the first entry. */ + for (uint64_t i = 0; i < si->master_fields_count; i++) + si->lp_ele = lpNext(si->lp,si->lp_ele); + } } else if (si->rev) { /* If we are itereating in the reverse order, and this is not * the first entry emitted for this listpack, then we already From 993716c351856f30487d6c8d9cc64423183c5f1b Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sun, 22 Jul 2018 21:16:00 +0300 Subject: [PATCH 026/616] Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR --- src/object.c | 29 +++++++++++++++++++++++++++-- src/scripting.c | 4 ++-- src/server.h | 1 + 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index 82b82632b..f72b29dd9 100644 --- a/src/object.c +++ b/src/object.c @@ -986,6 +986,18 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; + mem = 0; + mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + + dictSlots(server.lua_scripts) * sizeof(dictEntry*); + mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + + dictSlots(server.repl_scriptcache_dict) * sizeof(dictEntry*); + if (listLength(server.repl_scriptcache_fifo) > 0) { + mem += listLength(server.repl_scriptcache_fifo) * (sizeof(listNode) + + sdsZmallocSize(listNodeValue(listFirst(server.repl_scriptcache_fifo)))); + } + mh->lua_caches = mem; + mem_total+=mem; + for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; long long keyscount = dictSize(db->dict); @@ -1043,6 +1055,7 @@ sds getMemoryDoctorReport(void) { int high_alloc_rss = 0; /* High rss overhead. */ int big_slave_buf = 0; /* Slave buffers are too big. */ int big_client_buf = 0; /* Client buffers are too big. */ + int many_scripts = 0; /* Script cache has too many scripts. */ int num_reports = 0; struct redisMemOverhead *mh = getMemoryOverheadData(); @@ -1093,6 +1106,12 @@ sds getMemoryDoctorReport(void) { big_slave_buf = 1; num_reports++; } + + /* Too many (over 42) scripts are cached? */ + if (dictSize(server.lua_scripts) > 42) { + many_scripts = 1; + num_reports++; + } } sds s; @@ -1122,7 +1141,7 @@ sds getMemoryDoctorReport(void) { s = sdscatprintf(s," * High allocator RSS overhead: This instance has an RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the allocator is much larger than the sum what the allocator actually holds). This problem is usually due to a large peak memory (check if there is a peak memory entry above in the report), you can try the MEMORY PURGE command to reclaim it.\n\n"); } if (high_proc_rss) { - s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to LUA scripts or Modules.\n\n"); + s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to Lua scripts or Modules.\n\n"); } if (big_slave_buf) { s = sdscat(s," * Big slave buffers: The slave output buffers in this instance are greater than 10MB for each slave (on average). This likely means that there is some slave instance that is struggling receiving data, either because it is too slow or because of networking issues. As a result, data piles on the master output buffers. Please try to identify what slave is not receiving data correctly and why. You can use the INFO output in order to check the slaves delays and the CLIENT LIST command to check the output buffers of each slave.\n\n"); @@ -1130,6 +1149,9 @@ sds getMemoryDoctorReport(void) { if (big_client_buf) { s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n"); } + if (many_scripts) { + s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 42). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); + } s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n"); } freeMemoryOverheadData(mh); @@ -1236,7 +1258,7 @@ void memoryCommand(client *c) { } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { struct redisMemOverhead *mh = getMemoryOverheadData(); - addReplyMultiBulkLen(c,(24+mh->num_dbs)*2); + addReplyMultiBulkLen(c,(25+mh->num_dbs)*2); addReplyBulkCString(c,"peak.allocated"); addReplyLongLong(c,mh->peak_allocated); @@ -1259,6 +1281,9 @@ void memoryCommand(client *c) { addReplyBulkCString(c,"aof.buffer"); addReplyLongLong(c,mh->aof_buffer); + addReplyBulkCString(c,"lua.caches"); + addReplyLongLong(c,mh->lua_caches); + for (size_t j = 0; j < mh->num_dbs; j++) { char dbname[32]; snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid); diff --git a/src/scripting.c b/src/scripting.c index e131d8ae0..1a9cc1c60 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,7 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); - server.lua_scripts_mem = sizeof(dict); + server.lua_scripts_mem = 0; /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1209,7 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); - server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); + server.lua_scripts_mem += sdsZmallocSize(sha) + sdsZmallocSize(body->ptr); incrRefCount(body); return sha; } diff --git a/src/server.h b/src/server.h index 260ba80d7..3d9fa2307 100644 --- a/src/server.h +++ b/src/server.h @@ -834,6 +834,7 @@ struct redisMemOverhead { size_t clients_slaves; size_t clients_normal; size_t aof_buffer; + size_t lua_caches; size_t overhead_total; size_t dataset; size_t total_keys; From e6ea603ad3ae53d4d33c48ec94699912dcfac716 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 14:13:58 +0200 Subject: [PATCH 027/616] Dynamic HZ: separate hz from the configured hz. This way we can remember what the user configured HZ is, but change the actual HZ dynamically if needed in the dynamic HZ feature implementation. --- src/config.c | 16 ++++++++-------- src/server.c | 5 ++++- src/server.h | 3 +++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/config.c b/src/config.c index 54494c8e1..5acea06d5 100644 --- a/src/config.c +++ b/src/config.c @@ -441,9 +441,9 @@ void loadServerConfigFromString(char *config) { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - server.hz = atoi(argv[1]); - if (server.hz < CONFIG_MIN_HZ) server.hz = CONFIG_MIN_HZ; - if (server.hz > CONFIG_MAX_HZ) server.hz = CONFIG_MAX_HZ; + server.config_hz = atoi(argv[1]); + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { int yes; @@ -1158,11 +1158,11 @@ void configSetCommand(client *c) { } config_set_numerical_field( "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { } config_set_numerical_field( - "hz",server.hz,0,INT_MAX) { + "hz",server.config_hz,0,INT_MAX) { /* Hz is more an hint from the user, so we accept values out of range * but cap them to reasonable values. */ - if (server.hz < CONFIG_MIN_HZ) server.hz = CONFIG_MIN_HZ; - if (server.hz > CONFIG_MAX_HZ) server.hz = CONFIG_MAX_HZ; + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) @@ -1329,7 +1329,7 @@ void configGetCommand(client *c) { config_get_numerical_field("slave-announce-port",server.slave_announce_port); config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("hz",server.hz); + config_get_numerical_field("hz",server.config_hz); config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); @@ -2098,7 +2098,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",server.hz,CONFIG_DEFAULT_HZ); + rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); rewriteConfigYesNoOption(state,"rdb-save-incremental-fsync",server.rdb_save_incremental_fsync,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC); rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED); diff --git a/src/server.c b/src/server.c index 93c0a46fd..557af84d9 100644 --- a/src/server.c +++ b/src/server.c @@ -1512,7 +1512,7 @@ void initServerConfig(void) { server.timezone = timezone; /* Initialized by tzset(). */ server.configfile = NULL; server.executable = NULL; - server.hz = CONFIG_DEFAULT_HZ; + server.config_hz = CONFIG_DEFAULT_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = CONFIG_DEFAULT_SERVER_PORT; server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; @@ -1991,6 +1991,7 @@ void initServer(void) { server.syslog_facility); } + server.hz = server.config_hz; server.pid = getpid(); server.current_client = NULL; server.clients = listCreate(); @@ -3074,6 +3075,7 @@ sds genRedisInfoString(char *section) { "uptime_in_seconds:%jd\r\n" "uptime_in_days:%jd\r\n" "hz:%d\r\n" + "configured_hz:%d\r\n" "lru_clock:%ld\r\n" "executable:%s\r\n" "config_file:%s\r\n", @@ -3097,6 +3099,7 @@ sds genRedisInfoString(char *section) { (intmax_t)uptime, (intmax_t)(uptime/(3600*24)), server.hz, + server.config_hz, (unsigned long) lruclock, server.executable ? server.executable : "", server.configfile ? server.configfile : ""); diff --git a/src/server.h b/src/server.h index dae7561d8..4ac323da1 100644 --- a/src/server.h +++ b/src/server.h @@ -923,6 +923,9 @@ struct redisServer { char *configfile; /* Absolute config file path, or NULL */ char *executable; /* Absolute executable file path. */ char **exec_argv; /* Executable argv vector (copy). */ + int config_hz; /* Configured HZ value. May be different than + the actual 'hz' field value if dynamic-hz + is enabled. */ int hz; /* serverCron() calls frequency in hertz */ redisDb *db; dict *commands; /* Command table */ From b65ddfb16a7060a543b523feadeca1234cffd323 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 14:21:04 +0200 Subject: [PATCH 028/616] Dynamic HZ: adapt cron frequency to number of clients. --- src/server.c | 11 +++++++++++ src/server.h | 11 ++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index 557af84d9..aff7911e4 100644 --- a/src/server.c +++ b/src/server.c @@ -1096,6 +1096,17 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Update the time cache. */ updateCachedTime(); + /* Adapt the server.hz value to the number of configured clients. If we have + * many clients, we want to call serverCron() with an higher frequency. */ + server.hz = server.config_hz; + while (listLength(server.clients) / server.hz > MAX_CLIENTS_PER_CLOCK_TICK) { + server.hz *= 2; + if (server.hz > CONFIG_MAX_HZ) { + server.hz = CONFIG_MAX_HZ; + break; + } + } + run_with_period(100) { trackInstantaneousMetric(STATS_METRIC_COMMAND,server.stat_numcommands); trackInstantaneousMetric(STATS_METRIC_NET_INPUT, diff --git a/src/server.h b/src/server.h index 4ac323da1..a139b5440 100644 --- a/src/server.h +++ b/src/server.h @@ -78,12 +78,13 @@ typedef long long mstime_t; /* millisecond time type. */ #define C_ERR -1 /* Static server configuration */ -#define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ +#define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 #define CONFIG_MAX_HZ 500 -#define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port */ -#define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog */ -#define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* default client timeout: infinite */ +#define MAX_CLIENTS_PER_CLOCK_TICK 200 /* HZ is adapted based on that. */ +#define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port. */ +#define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog. */ +#define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* Default client timeout: infinite */ #define CONFIG_DEFAULT_DBNUM 16 #define CONFIG_MAX_LINE 1024 #define CRON_DBS_PER_CALL 16 @@ -91,7 +92,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define PROTO_SHARED_SELECT_CMDS 10 #define OBJ_SHARED_INTEGERS 10000 #define OBJ_SHARED_BULKHDR_LEN 32 -#define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages */ +#define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages.*/ #define AOF_REWRITE_PERC 100 #define AOF_REWRITE_MIN_SIZE (64*1024*1024) #define AOF_REWRITE_ITEMS_PER_CMD 64 From 9a375e5408e3539e4ed8abc2f20067c84ef8f13e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 18:44:38 +0200 Subject: [PATCH 029/616] Change 42 to 1000 as warning level for cached scripts. Related to #4883. --- src/object.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object.c b/src/object.c index c1f6ac4b2..406816140 100644 --- a/src/object.c +++ b/src/object.c @@ -1138,8 +1138,8 @@ sds getMemoryDoctorReport(void) { num_reports++; } - /* Too many (over 42) scripts are cached? */ - if (dictSize(server.lua_scripts) > 42) { + /* Too many scripts are cached? */ + if (dictSize(server.lua_scripts) > 1000) { many_scripts = 1; num_reports++; } @@ -1181,7 +1181,7 @@ sds getMemoryDoctorReport(void) { s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n"); } if (many_scripts) { - s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 42). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); + s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 1000). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); } s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n"); } From 1daee8c74859a9980e9a055fe9d788143293c81e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 00:18:39 +0200 Subject: [PATCH 030/616] string2ll(): test for NULL pointer in all the cases. --- src/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index ca5435026..02298e8df 100644 --- a/src/util.c +++ b/src/util.c @@ -369,7 +369,7 @@ int string2ll(const char *s, size_t slen, long long *value) { v = p[0]-'0'; p++; plen++; } else if (p[0] == '0' && slen == 1) { - *value = 0; + if (value != NULL) *value = 0; return 1; } else { return 0; From e8f5d2194020886b39da19fc98b8a77ede516256 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 00:20:20 +0200 Subject: [PATCH 031/616] string2ll(): remove duplicated check for special case. Related to #5157. The PR author correctly indentified that the check was duplicated, but removing the second one introduces a bug that was fixed in the past (hence the duplication). Instead we can remove the first instance of the check without issues. --- src/util.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/util.c b/src/util.c index 02298e8df..e2ada556a 100644 --- a/src/util.c +++ b/src/util.c @@ -349,12 +349,7 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen == slen) return 0; - /* Special case: first and only digit is 0. */ - if (slen == 1 && p[0] == '0') { - if (value != NULL) *value = 0; - return 1; - } - + /* Handle negative integers. */ if (p[0] == '-') { negative = 1; p++; plen++; From b28cbe90e1205ba3c1fe072b09f6407fd5537730 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 10:22:12 +0200 Subject: [PATCH 032/616] Restore string2ll() to original version. See PR #5157. --- src/util.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/util.c b/src/util.c index e2ada556a..ca5435026 100644 --- a/src/util.c +++ b/src/util.c @@ -349,7 +349,12 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen == slen) return 0; - /* Handle negative integers. */ + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return 1; + } + if (p[0] == '-') { negative = 1; p++; plen++; @@ -364,7 +369,7 @@ int string2ll(const char *s, size_t slen, long long *value) { v = p[0]-'0'; p++; plen++; } else if (p[0] == '0' && slen == 1) { - if (value != NULL) *value = 0; + *value = 0; return 1; } else { return 0; From d4ae76d1a6cde03c957000f870ff3d1c0e8c3fb9 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 1 Apr 2018 17:09:35 +0300 Subject: [PATCH 033/616] fix slave buffer test suite false positives it looks like on slow machines we're getting: [err]: slave buffer are counted correctly in tests/unit/maxmemory.tcl Expected condition '$slave_buf > 2*1024*1024' to be true (16914 > 2*1024*1024) this is a result of the slave waking up too early and eating the slave buffer before the traffic and the test ends. --- tests/unit/maxmemory.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index e3cd11114..7629fe05e 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -182,7 +182,7 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { # put the slave to sleep set rd_slave [redis_deferring_client] - $rd_slave debug sleep 60 + $rd_slave debug sleep 300 # send some 10mb woth of commands that don't increase the memory usage if {$pipeline == 1} { From f4ac796c3476ffa777a95a9795171ac8dc56dc06 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 10:27:20 +0200 Subject: [PATCH 034/616] string2ll(): better commenting. --- src/util.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/util.c b/src/util.c index dd776162e..3fa6c9244 100644 --- a/src/util.c +++ b/src/util.c @@ -346,6 +346,7 @@ int string2ll(const char *s, size_t slen, long long *value) { int negative = 0; unsigned long long v; + /* A zero length string is not a valid number. */ if (plen == slen) return 0; @@ -355,6 +356,8 @@ int string2ll(const char *s, size_t slen, long long *value) { return 1; } + /* Handle negative numbers: just set a flag and continue like if it + * was a positive number. Later convert into negative. */ if (p[0] == '-') { negative = 1; p++; plen++; @@ -372,6 +375,7 @@ int string2ll(const char *s, size_t slen, long long *value) { return 0; } + /* Parse all the other digits, checking for overflow at every step. */ while (plen < slen && p[0] >= '0' && p[0] <= '9') { if (v > (ULLONG_MAX / 10)) /* Overflow. */ return 0; @@ -388,6 +392,8 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen < slen) return 0; + /* Convert to negative if needed, and do the final overflow check when + * converting from unsigned long long to long long. */ if (negative) { if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ return 0; From 8513f31be06137dd4fe8ad0f7ec5cb84b0503cb5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 11:07:27 +0200 Subject: [PATCH 035/616] Streams: refactoring of next entry seek in the iterator. After #5161 the code could be made a bit more obvious for newcomers. --- src/t_stream.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 15a41bedf..4566b9bf3 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -548,15 +548,19 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { si->master_fields_count = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ si->master_fields_start = si->lp_ele; - /* We are now pointing the start filed of the master entry. If - * we are iterating in reverse order, we need to seek the - * end of the listpack. */ - if (si->rev) { - si->lp_ele = lpLast(si->lp); - } else { - /* Skip master fileds to seek the first entry. */ + /* We are now pointing to the first field of the master entry. + * We need to seek either the first or the last entry depending + * on the direction of the iteration. */ + if (!si->rev) { + /* If we are iterating in normal order, skip the master fields + * to seek the first actual entry. */ for (uint64_t i = 0; i < si->master_fields_count; i++) si->lp_ele = lpNext(si->lp,si->lp_ele); + } else { + /* If we are iterating in reverse direction, just seek the + * last part of the last entry in the listpack (that is, the + * fields count). */ + si->lp_ele = lpLast(si->lp); } } else if (si->rev) { /* If we are itereating in the reverse order, and this is not From 53d46fa712619af7bcd7883aabe279344336be11 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 17:27:43 +0200 Subject: [PATCH 036/616] Make changes of PR #5154 hopefully simpler. --- src/cluster.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 5085c2a6a..19688d593 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5146,7 +5146,10 @@ try_again: serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); } - int num_keys_non_expired = 0; + int expired = 0; /* Number of keys that we'll find already expired. + Note that serializing large keys may take some time + so certain keys that were found non expired by the + lookupKey() function, may be expired later. */ /* Create RESTORE payload and generate the protocol to call the command. */ for (j = 0; j < num_keys; j++) { @@ -5155,10 +5158,12 @@ try_again: if (expireat != -1) { ttl = expireat-mstime(); - if (ttl < 0) continue; + if (ttl < 0) { + expired++; + continue; + } if (ttl < 1) ttl = 1; } - num_keys_non_expired++; serverAssertWithInfo(c,NULL, rioWriteBulkCount(&cmd,'*',replace ? 5 : 4)); @@ -5221,9 +5226,9 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ - if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys_non_expired+1)); + if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); - for (j = 0; j < num_keys_non_expired; j++) { + for (j = 0; j < num_keys-expired; j++) { if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; From 0bdeb861f48b4bed269e80d9211c0ce854465a4a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 17:31:39 +0200 Subject: [PATCH 037/616] Example the magic +1 in migrateCommand(). Related to #5154. --- src/cluster.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cluster.c b/src/cluster.c index 19688d593..8b76d9ad0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5226,6 +5226,10 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ + /* Allocate the new argument vector that will replace the current command, + * to propagate the MIGRATE as a DEL command (if no COPY option was given). + * We allocate num_keys+1 because the additional argument is for "DEL" + * command name itself. */ if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); for (j = 0; j < num_keys-expired; j++) { From eb0e5fe4e3b31c2b6cfebbe95324ce488211d372 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 25 Jul 2018 00:06:27 +0800 Subject: [PATCH 038/616] Streams: fix xdel memory leak --- src/t_stream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/t_stream.c b/src/t_stream.c index 7b1076b16..5a5263325 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -757,6 +757,7 @@ int streamDeleteItem(stream *s, streamID *id) { streamIteratorRemoveEntry(&si,&myid); deleted = 1; } + streamIteratorStop(&si); return deleted; } From dd071ff6c95cbb7427e5f0c3a6ca84e3c672d87a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 25 Jul 2018 18:13:34 +0800 Subject: [PATCH 039/616] optimize flushdb, avoid useless loops --- src/db.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index 055af71be..39686f1b6 100644 --- a/src/db.c +++ b/src/db.c @@ -329,7 +329,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * database(s). Otherwise -1 is returned in the specific case the * DB number is out of range, and errno is set to EINVAL. */ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { - int j, async = (flags & EMPTYDB_ASYNC); + int async = (flags & EMPTYDB_ASYNC); long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -337,7 +337,10 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } - for (j = 0; j < server.dbnum; j++) { + int j = dbnum == -1 ? 0 : dbnum; + int dbmax = dbnum == -1 ? server.dbnum : dbnum+1; + + for (; j < dbmax; j++) { if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { From 1c2352f0dd02753a3476871cec0dbb6a4b032064 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Jul 2018 16:32:52 +0200 Subject: [PATCH 040/616] Make emptyDb() change introduced in #4852 simpler to read. --- src/db.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 39686f1b6..a2890639b 100644 --- a/src/db.c +++ b/src/db.c @@ -337,10 +337,15 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } - int j = dbnum == -1 ? 0 : dbnum; - int dbmax = dbnum == -1 ? server.dbnum : dbnum+1; + int startdb, enddb; + if (dbnum == -1) { + startdb = 0; + enddb = server.dbnum-1; + } else { + startdb = enddb = dbnum; + } - for (; j < dbmax; j++) { + for (int j = startdb; j <= enddb; j++) { if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { From fd174cca23ede115ff71b379f10b407cfb54d3ad Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Jul 2018 16:34:46 +0200 Subject: [PATCH 041/616] Remove useless conditional from emptyDb(). Related to #4852. --- src/db.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/db.c b/src/db.c index a2890639b..6efe47011 100644 --- a/src/db.c +++ b/src/db.c @@ -346,7 +346,6 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } for (int j = startdb; j <= enddb; j++) { - if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { emptyDbAsync(&server.db[j]); From 4a30adde31dec0d54c38eb88a5af64a1466f6744 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 29 Jul 2018 10:08:47 +0300 Subject: [PATCH 042/616] add DEBUG LOG, to to assist test suite debugging --- src/debug.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/debug.c b/src/debug.c index 32be3c59c..a66390dbb 100644 --- a/src/debug.c +++ b/src/debug.c @@ -290,6 +290,7 @@ void debugCommand(client *c) { "CRASH-AND-RECOVER -- Hard crash and restart after delay.", "DIGEST -- Output a hex signature representing the current DB content.", "ERROR -- Return a Redis protocol error with as message. Useful for clients unit tests to simulate Redis errors.", +"LOG -- write message to the server log.", "HTSTATS -- Return hash table statistics of the specified Redis database.", "HTSTATS-KEY -- Like htstats but for the hash table stored as key's value.", "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.", @@ -333,6 +334,9 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"assert")) { if (c->argc >= 3) c->argv[2] = tryObjectEncoding(c->argv[2]); serverAssertWithInfo(c,c->argv[0],1 == 2); + } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) { + serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); From c426d85c4c877a21c959b306b81d8df46259cbd6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:37:30 +0200 Subject: [PATCH 043/616] Control dynamic HZ via server configuration. --- src/server.c | 17 +++++++++++------ src/server.h | 2 ++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/server.c b/src/server.c index 965cae967..23450c8fd 100644 --- a/src/server.c +++ b/src/server.c @@ -1096,14 +1096,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Update the time cache. */ updateCachedTime(); + server.hz = server.config_hz; /* Adapt the server.hz value to the number of configured clients. If we have * many clients, we want to call serverCron() with an higher frequency. */ - server.hz = server.config_hz; - while (listLength(server.clients) / server.hz > MAX_CLIENTS_PER_CLOCK_TICK) { - server.hz *= 2; - if (server.hz > CONFIG_MAX_HZ) { - server.hz = CONFIG_MAX_HZ; - break; + if (server.dynamic_hz) { + while (listLength(server.clients) / server.hz > + MAX_CLIENTS_PER_CLOCK_TICK) + { + server.hz *= 2; + if (server.hz > CONFIG_MAX_HZ) { + server.hz = CONFIG_MAX_HZ; + break; + } } } @@ -1524,6 +1528,7 @@ void initServerConfig(void) { server.configfile = NULL; server.executable = NULL; server.config_hz = CONFIG_DEFAULT_HZ; + server.dynamic_hz = CONFIG_DEFAULT_DYNAMIC_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = CONFIG_DEFAULT_SERVER_PORT; server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; diff --git a/src/server.h b/src/server.h index 4d63faa66..21e4c4279 100644 --- a/src/server.h +++ b/src/server.h @@ -78,6 +78,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define C_ERR -1 /* Static server configuration */ +#define CONFIG_DEFAULT_DYNAMIC_HZ 1 /* Adapt hz to # of clients.*/ #define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 #define CONFIG_MAX_HZ 500 @@ -925,6 +926,7 @@ struct redisServer { char *configfile; /* Absolute config file path, or NULL */ char *executable; /* Absolute executable file path. */ char **exec_argv; /* Executable argv vector (copy). */ + int dynamic_hz; /* Change hz value depending on # of clients. */ int config_hz; /* Configured HZ value. May be different than the actual 'hz' field value if dynamic-hz is enabled. */ From be28050ac02b4154ba937a6487c0666d4d4de856 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:44:52 +0200 Subject: [PATCH 044/616] Make dynamic hz actually configurable. --- src/config.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config.c b/src/config.c index 5acea06d5..7bd9592b2 100644 --- a/src/config.c +++ b/src/config.c @@ -440,6 +440,10 @@ void loadServerConfigFromString(char *config) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"dynamic-hz") && argc == 2) { + if ((server.dynamic_hz = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"hz") && argc == 2) { server.config_hz = atoi(argv[1]); if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; @@ -1072,6 +1076,8 @@ void configSetCommand(client *c) { "slave-lazy-flush",server.repl_slave_lazy_flush) { } config_set_bool_field( "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { + } config_set_bool_field( + "dynamic-hz",server.dynamic_hz) { /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ @@ -1375,6 +1381,8 @@ void configGetCommand(client *c) { server.lazyfree_lazy_server_del); config_get_bool_field("slave-lazy-flush", server.repl_slave_lazy_flush); + config_get_bool_field("dynamic-hz", + server.dynamic_hz); /* Enum values */ config_get_enum_field("maxmemory-policy", @@ -2108,6 +2116,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); rewriteConfigYesNoOption(state,"slave-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); + rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); From 15be174cf1c601793477f0d5ec71da1646fe09e5 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:50:15 +0200 Subject: [PATCH 045/616] Document dynamic-hz in the example redis.conf. --- redis.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/redis.conf b/redis.conf index 42d24f26e..5cbc74bbd 100644 --- a/redis.conf +++ b/redis.conf @@ -1205,6 +1205,22 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporary raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used as +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + # When a child rewrites the AOF file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid From acaa18f1d1b3d0b74fad4c6bd6a2b034da6cea95 Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Mon, 30 Jul 2018 16:18:56 +0300 Subject: [PATCH 046/616] Few typo fixes --- src/module.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/module.c b/src/module.c index 9809cd74e..3735cbe24 100644 --- a/src/module.c +++ b/src/module.c @@ -548,7 +548,7 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { ctx->keys_pos[ctx->keys_count++] = pos; } -/* Helper for RM_CreateCommand(). Truns a string representing command +/* Helper for RM_CreateCommand(). Turns a string representing command * flags into the command flags used by the Redis core. * * It returns the set of flags, or -1 if unknown flags are found. */ @@ -595,7 +595,7 @@ int commandFlagsFromString(char *s) { * And is supposed to always return REDISMODULE_OK. * * The set of flags 'strflags' specify the behavior of the command, and should - * be passed as a C string compoesd of space separated words, like for + * be passed as a C string composed of space separated words, like for * example "write deny-oom". The set of flags are: * * * **"write"**: The command may modify the data set (it may also read @@ -616,7 +616,7 @@ int commandFlagsFromString(char *s) { * * **"allow-stale"**: The command is allowed to run on slaves that don't * serve stale data. Don't use if you don't know what * this means. - * * **"no-monitor"**: Don't propoagate the command on monitor. Use this if + * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensible data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or @@ -956,9 +956,9 @@ RedisModuleString *moduleAssertUnsharedString(RedisModuleString *str) { return str; } -/* Append the specified buffere to the string 'str'. The string must be a +/* Append the specified buffer to the string 'str'. The string must be a * string created by the user that is referenced only a single time, otherwise - * REDISMODULE_ERR is returend and the operation is not performed. */ + * REDISMODULE_ERR is returned and the operation is not performed. */ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len) { UNUSED(ctx); str = moduleAssertUnsharedString(str); @@ -1118,7 +1118,7 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { * * Note that in the above example there is no reason to postpone the array * length, since we produce a fixed number of elements, but in the practice - * the code may use an interator or other ways of creating the output so + * the code may use an iterator or other ways of creating the output so * that is not easy to calculate in advance the number of elements. */ void RM_ReplySetArrayLength(RedisModuleCtx *ctx, long len) { @@ -1410,7 +1410,7 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { * to call other APIs with the key handle as argument to perform * operations on the key. * - * The return value is the handle repesenting the key, that must be + * The return value is the handle representing the key, that must be * closed with RM_CloseKey(). * * If the key does not exist and WRITE mode is requested, the handle @@ -1664,7 +1664,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { * Key API for List type * -------------------------------------------------------------------------- */ -/* Push an element into a list, on head or tail depending on 'where' argumnet. +/* Push an element into a list, on head or tail depending on 'where' argument. * If the key pointer is about an empty key opened for writing, the key * is created. On error (key opened for read-only operations or of the wrong * type) REDISMODULE_ERR is returned, otherwise REDISMODULE_OK is returned. */ @@ -1769,7 +1769,7 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f * The input and output flags, and the return value, have the same exact * meaning, with the only difference that this function will return * REDISMODULE_ERR even when 'score' is a valid double number, but adding it - * to the existing score resuts into a NaN (not a number) condition. + * to the existing score results into a NaN (not a number) condition. * * This function has an additional field 'newscore', if not NULL is filled * with the new score of the element after the increment, if no error @@ -3793,11 +3793,11 @@ void moduleReleaseGIL(void) { * -------------------------------------------------------------------------- */ /* Subscribe to keyspace notifications. This is a low-level version of the - * keyspace-notifications API. A module cand register callbacks to be notified + * keyspace-notifications API. A module can register callbacks to be notified * when keyspce events occur. * * Notification events are filtered by their type (string events, set events, - * etc), and the subsriber callback receives only events that match a specific + * etc), and the subscriber callback receives only events that match a specific * mask of event types. * * When subscribing to notifications with RedisModule_SubscribeToKeyspaceEvents @@ -3832,7 +3832,7 @@ void moduleReleaseGIL(void) { * used to send anything to the client, and has the db number where the event * occurred as its selected db number. * - * Notice that it is not necessary to enable norifications in redis.conf for + * Notice that it is not necessary to enable notifications in redis.conf for * module notifications to work. * * Warning: the notification callbacks are performed in a synchronous manner, @@ -3876,7 +3876,7 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid) ctx.client = moduleKeyspaceSubscribersClient; selectDb(ctx.client, dbid); - /* mark the handler as activer to avoid reentrant loops. + /* mark the handler as active to avoid reentrant loops. * If the subscriber performs an action triggering itself, * it will not be notified about it. */ sub->active = 1; From d6f5ec6f098f7f6b741bc3face1608c25b233caa Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Mon, 30 Jul 2018 21:32:07 +0800 Subject: [PATCH 047/616] Streams: add mmid_supp argument in streamParseIDOrReply(). If 'mmid_supp' is set to 0, "-" and "+" will be treated as an invalid ID. --- src/t_stream.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index ea1290faf..8725542c9 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1060,14 +1060,18 @@ int string2ull(const char *s, unsigned long long *value) { * form, just stating the milliseconds time part of the stream. In such a case * the missing part is set according to the value of 'missing_seq' parameter. * The IDs "-" and "+" specify respectively the minimum and maximum IDs - * that can be represented. + * that can be represented. If 'mmid_supp' is set to 0, "-" and "+" will be + * treated as an invalid ID. * * If 'c' is set to NULL, no reply is sent to the client. */ -int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { +int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int mmid_supp) { char buf[128]; if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid; memcpy(buf,o->ptr,sdslen(o->ptr)+1); + if (!mmid_supp && (buf[0] == '-' || buf[0] == '+') && + buf[1] == '\0') goto invalid; + /* Handle the "-" and "+" special cases. */ if (buf[0] == '-' && buf[1] == '\0') { id->ms = 0; From 6491717c88143507567420398131eaa8933f3ec9 Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Mon, 30 Jul 2018 21:33:01 +0800 Subject: [PATCH 048/616] Streams: rearrange the usage of '-' and '+' IDs in stream commands. --- src/t_stream.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 8725542c9..f233785ae 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1137,7 +1137,7 @@ void xaddCommand(client *c) { maxlen_arg_idx = i; } else { /* If we are here is a syntax error or a valid ID. */ - if (streamParseIDOrReply(c,c->argv[i],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[i],&id,0,0) != C_OK) return; id_given = 1; break; } @@ -1206,8 +1206,8 @@ void xrangeGenericCommand(client *c, int rev) { robj *startarg = rev ? c->argv[3] : c->argv[2]; robj *endarg = rev ? c->argv[2] : c->argv[3]; - if (streamParseIDOrReply(c,startarg,&startid,0) == C_ERR) return; - if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX) == C_ERR) return; + if (streamParseIDOrReply(c,startarg,&startid,0,1) == C_ERR) return; + if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX,1) == C_ERR) return; /* Parse the COUNT option if any. */ if (c->argc > 4) { @@ -1393,7 +1393,7 @@ void xreadCommand(client *c) { ids[id_idx].seq = UINT64_MAX; continue; } - if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK) + if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0,0) != C_OK) goto cleanup; } @@ -1667,7 +1667,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0,0) != C_OK) { return; } streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); @@ -1684,7 +1684,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0,1) != C_OK) { return; } cg->last_id = id; @@ -1744,7 +1744,7 @@ void xackCommand(client *c) { for (int j = 3; j < c->argc; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL: it will have a reference to the @@ -1791,9 +1791,9 @@ void xpendingCommand(client *c) { if (c->argc >= 6) { if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR) + if (streamParseIDOrReply(c,c->argv[3],&startid,0,1) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR) + if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX,1) == C_ERR) return; } @@ -2002,7 +2002,7 @@ void xclaimCommand(client *c) { int j; for (j = 4; j < c->argc; j++) { streamID id; - if (streamParseIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; + if (streamParseIDOrReply(NULL,c->argv[j],&id,0,0) != C_OK) break; } int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */ @@ -2061,7 +2061,7 @@ void xclaimCommand(client *c) { for (int j = 5; j <= last_id_arg; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL. */ @@ -2144,13 +2144,13 @@ void xdelCommand(client *c) { * executed because at some point an invalid ID is parsed. */ streamID id; for (int j = 2; j < c->argc; j++) { - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; } /* Actually apply the command. */ int deleted = 0; for (int j = 2; j < c->argc; j++) { - streamParseIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */ + streamParseIDOrReply(c,c->argv[j],&id,0,0); /* Retval already checked. */ deleted += streamDeleteItem(s,&id); } From 3c19ae941d17dee7f89c282946bc379ba2106b09 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 17:42:30 +0200 Subject: [PATCH 049/616] Add year in log. User: "is there a reason why redis server logs are missing the year in the "date time"?" Me: "I guess I did not imagine it would be stable enough to run for several years". --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 23450c8fd..3b2e8d2df 100644 --- a/src/server.c +++ b/src/server.c @@ -357,7 +357,7 @@ void serverLogRaw(int level, const char *msg) { gettimeofday(&tv,NULL); struct tm tm; nolocks_localtime(&tm,tv.tv_sec,server.timezone,server.daylight_active); - off = strftime(buf,sizeof(buf),"%d %b %H:%M:%S.",&tm); + off = strftime(buf,sizeof(buf),"%d %b %Y %H:%M:%S.",&tm); snprintf(buf+off,sizeof(buf)-off,"%03d",(int)tv.tv_usec/1000); if (server.sentinel_mode) { role_char = 'X'; /* Sentinel. */ From 1e394b73b799837f23a4f2a9d15366e1cd23fcf9 Mon Sep 17 00:00:00 2001 From: Pavel Rochnyack Date: Wed, 25 Jul 2018 21:50:24 +0700 Subject: [PATCH 050/616] INFO CPU: higher precision of reported values Closes: #5033 --- src/server.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server.c b/src/server.c index 3b2e8d2df..c0ea66282 100644 --- a/src/server.c +++ b/src/server.c @@ -3536,14 +3536,14 @@ sds genRedisInfoString(char *section) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# CPU\r\n" - "used_cpu_sys:%.2f\r\n" - "used_cpu_user:%.2f\r\n" - "used_cpu_sys_children:%.2f\r\n" - "used_cpu_user_children:%.2f\r\n", - (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000, - (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000, - (float)c_ru.ru_stime.tv_sec+(float)c_ru.ru_stime.tv_usec/1000000, - (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000); + "used_cpu_sys:%ld.%06ld\r\n" + "used_cpu_user:%ld.%06ld\r\n" + "used_cpu_sys_children:%ld.%06ld\r\n" + "used_cpu_user_children:%ld.%06ld\r\n", + (long)self_ru.ru_stime.tv_sec, (long)self_ru.ru_stime.tv_usec, + (long)self_ru.ru_utime.tv_sec, (long)self_ru.ru_utime.tv_usec, + (long)c_ru.ru_stime.tv_sec, (long)c_ru.ru_stime.tv_usec, + (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); } /* Command statistics */ From 4ebb4d54c69f24360626bd9b458e8678c9c01de8 Mon Sep 17 00:00:00 2001 From: Salvatore Sanfilippo Date: Mon, 30 Jul 2018 18:03:15 +0200 Subject: [PATCH 051/616] Merge pull request #5168 from rpv-tomsk/issue-5033 INFO CPU: higher precision of reported values From 782928769187c977252c9f698535d824ac5409de Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 30 Jul 2018 16:43:21 +0300 Subject: [PATCH 052/616] test suite conveniency improvements * allowing --single to be repeated * adding --only so that only a specific test inside a unit can be run * adding --skiptill useful to resume a test that crashed passed the problematic unit. useful together with --clients 1 * adding --skipfile to use a file containing list of tests names to skip * printing the names of the tests that are skiped by skipfile or denytags * adding --config to add config file options from command line --- runtest | 2 +- tests/support/test.tcl | 20 ++++++++++++++ tests/test_helper.tcl | 60 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/runtest b/runtest index d8451df57..ade1bd09a 100755 --- a/runtest +++ b/runtest @@ -11,4 +11,4 @@ then echo "You need tcl 8.5 or newer in order to run the Redis test" exit 1 fi -$TCLSH tests/test_helper.tcl $* +$TCLSH tests/test_helper.tcl "${@}" diff --git a/tests/support/test.tcl b/tests/support/test.tcl index d60eb3c47..6f02f2f12 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -1,6 +1,8 @@ set ::num_tests 0 set ::num_passed 0 set ::num_failed 0 +set ::num_skipped 0 +set ::num_aborted 0 set ::tests_failed {} proc fail {msg} { @@ -68,10 +70,26 @@ proc test {name code {okpattern undefined}} { # abort if tagged with a tag to deny foreach tag $::denytags { if {[lsearch $::tags $tag] >= 0} { + incr ::num_aborted + send_data_packet $::test_server_fd ignore $name return } } + # abort if test name in skiptests + if {[lsearch $::skiptests $name] >= 0} { + incr ::num_skipped + send_data_packet $::test_server_fd skip $name + return + } + + # abort if test name in skiptests + if {[llength $::only_tests] > 0 && [lsearch $::only_tests $name] < 0} { + incr ::num_skipped + send_data_packet $::test_server_fd skip $name + return + } + # check if tagged with at least 1 tag to allow when there *is* a list # of tags to allow, because default policy is to run everything if {[llength $::allowtags] > 0} { @@ -82,6 +100,8 @@ proc test {name code {okpattern undefined}} { } } if {$matched < 1} { + incr ::num_aborted + send_data_packet $::test_server_fd ignore $name return } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 720a428a6..ba3dce71c 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -74,7 +74,11 @@ set ::stack_logging 0 set ::verbose 0 set ::quiet 0 set ::denytags {} +set ::skiptests {} set ::allowtags {} +set ::only_tests {} +set ::single_tests {} +set ::skip_till "" set ::external 0; # If "1" this means, we are running against external instance set ::file ""; # If set, runs only the tests in this comma separated list set ::curfile ""; # Hold the filename of the current suite @@ -255,6 +259,8 @@ proc accept_test_clients {fd addr port} { # testing: just used to signal that a given test started. # ok: a test was executed with success. # err: a test was executed with an error. +# skip: a test was skipped by skipfile or individual test options. +# ignore: a test was skipped by a group tag. # exception: there was a runtime exception while executing the test. # done: all the specified test file was processed, this test client is # ready to accept a new task. @@ -283,6 +289,14 @@ proc read_from_test_client fd { puts "\[[colorstr green $status]\]: $data" } set ::active_clients_task($fd) "(OK) $data" + } elseif {$status eq {skip}} { + if {!$::quiet} { + puts "\[[colorstr yellow $status]\]: $data" + } + } elseif {$status eq {ignore}} { + if {!$::quiet} { + puts "\[[colorstr cyan $status]\]: $data" + } } elseif {$status eq {err}} { set err "\[[colorstr red $status]\]: $data" puts $err @@ -412,11 +426,15 @@ proc print_help_screen {} { "--stack-logging Enable OSX leaks/malloc stack logging." "--accurate Run slow randomized tests for more iterations." "--quiet Don't show individual tests." - "--single Just execute the specified unit (see next option)." + "--single Just execute the specified unit (see next option). this option can be repeated." "--list-tests List all the available test units." + "--only Just execute the specified test by test name. this option can be repeated." + "--skiptill Skip all units until (and including) the specified one." "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." + "--config extra config file argument" + "--skipfile name of a file containing test names that should be skipped (one per line)" "--dont-clean don't delete redis log files after the run" "--wait-server wait after server is started (so that you can attach a debugger)" "--help Print this help screen." @@ -436,6 +454,18 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } incr j + } elseif {$opt eq {--config}} { + set arg2 [lindex $argv [expr $j+2]] + lappend ::global_overrides $arg + lappend ::global_overrides $arg2 + incr j + incr j + } elseif {$opt eq {--skipfile}} { + incr j + set fp [open $arg r] + set file_data [read $fp] + close $fp + set ::skiptests [split $file_data "\n"] } elseif {$opt eq {--valgrind}} { set ::valgrind 1 } elseif {$opt eq {--stack-logging}} { @@ -456,7 +486,13 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--force-failure}} { set ::force_failure 1 } elseif {$opt eq {--single}} { - set ::all_tests $arg + lappend ::single_tests $arg + incr j + } elseif {$opt eq {--only}} { + lappend ::only_tests $arg + incr j + } elseif {$opt eq {--skiptill}} { + set ::skip_till $arg incr j } elseif {$opt eq {--list-tests}} { foreach t $::all_tests { @@ -488,6 +524,26 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +if {$::skip_till != ""} { + set skipping 1 + foreach t $::all_tests { + if {$skipping == 0} { + lappend ::single_tests $t + } + if {$t == $::skip_till} { + set skipping 0 + } + } + if {$skipping} { + puts "test $::skip_till not found" + exit 0 + } +} + +if {[llength $::single_tests] > 0} { + set ::all_tests $::single_tests +} + proc attach_to_replication_stream {} { set s [socket [srv 0 "host"] [srv 0 "port"]] fconfigure $s -translation binary From fddeeae724bec1a8e1a3dbb2f18e608b79c2ddfc Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 31 Jul 2018 12:07:57 +0800 Subject: [PATCH 053/616] refactor dbOverwrite to make lazyfree work --- src/db.c | 17 +++++++++++------ src/lazyfree.c | 11 +++++++++++ src/server.h | 1 + src/t_set.c | 10 ++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/db.c b/src/db.c index 6efe47011..ec92a2b4e 100644 --- a/src/db.c +++ b/src/db.c @@ -184,14 +184,19 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { dictEntry *de = dictFind(db->dict,key->ptr); serverAssertWithInfo(NULL,key,de != NULL); + dictEntry auxentry = *de; + robj *old = dictGetVal(de); if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { - robj *old = dictGetVal(de); - int saved_lru = old->lru; - dictReplace(db->dict, key->ptr, val); - val->lru = saved_lru; - } else { - dictReplace(db->dict, key->ptr, val); + val->lru = old->lru; } + dictSetVal(db->dict, de, val); + + if (server.lazyfree_lazy_server_del) { + freeObjAsync(old); + dictSetVal(db->dict, &auxentry, NULL); + } + + dictFreeVal(db->dict, &auxentry); } /* High level Set operation. This function can be used in order to set diff --git a/src/lazyfree.c b/src/lazyfree.c index ac8a6bee9..3d3159c90 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -90,6 +90,17 @@ int dbAsyncDelete(redisDb *db, robj *key) { } } +/* Free an object, if the object is huge enough, free it in async way. */ +void freeObjAsync(robj *o) { + size_t free_effort = lazyfreeGetFreeEffort(o); + if (free_effort > LAZYFREE_THRESHOLD && o->refcount == 1) { + atomicIncr(lazyfree_objects,1); + bioCreateBackgroundJob(BIO_LAZY_FREE,o,NULL,NULL); + } else { + decrRefCount(o); + } +} + /* Empty a Redis DB asynchronously. What the function does actually is to * create a new empty set of hash tables and scheduling the old ones for * lazy freeing. */ diff --git a/src/server.h b/src/server.h index 14aed7fc0..a7985a39d 100644 --- a/src/server.h +++ b/src/server.h @@ -1824,6 +1824,7 @@ int dbAsyncDelete(redisDb *db, robj *key); void emptyDbAsync(redisDb *db); void slotToKeyFlushAsync(void); size_t lazyfreeGetPendingObjectsCount(void); +void freeObjAsync(robj *o); /* API to get key arguments from commands */ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); diff --git a/src/t_set.c b/src/t_set.c index 8f21f71b1..5d48e3c43 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -516,11 +516,7 @@ void spopWithCountCommand(client *c) { sdsfree(sdsele); } - /* Assign the new set as the key value. */ - incrRefCount(set); /* Protect the old set value. */ - dbOverwrite(c->db,c->argv[1],newset); - - /* Tranfer the old set to the client and release it. */ + /* Tranfer the old set to the client. */ setTypeIterator *si; si = setTypeInitIterator(set); while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) { @@ -539,7 +535,9 @@ void spopWithCountCommand(client *c) { decrRefCount(objele); } setTypeReleaseIterator(si); - decrRefCount(set); + + /* Assign the new set as the key value. */ + dbOverwrite(c->db,c->argv[1],newset); } /* Don't propagate the command itself even if we incremented the From 35ca6700600f2b1991e71e6196ec03ba4bad997f Mon Sep 17 00:00:00 2001 From: shenlongxing Date: Tue, 31 Jul 2018 16:01:44 +0800 Subject: [PATCH 054/616] Fix cluster-announce-ip memory leak --- src/cluster.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 8b76d9ad0..7a6a5f782 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3325,10 +3325,11 @@ void clusterCron(void) { if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; if (changed) { - prev_ip = curr_ip; - if (prev_ip) prev_ip = zstrdup(prev_ip); + if (prev_ip) zfree(prev_ip); + prev_ip = curr_ip; if (curr_ip) { + prev_ip = zstrdup(prev_ip); strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN); myself->ip[NET_IP_STR_LEN-1] = '\0'; } else { From 5401fe7fb948f6071d5f804fc3c1fbac77286dd5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:09:38 +0200 Subject: [PATCH 055/616] Introduce writeCommandsDeniedByDiskError(). --- src/server.c | 19 +++++++++++++++++++ src/server.h | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/src/server.c b/src/server.c index c0ea66282..2e0c36014 100644 --- a/src/server.c +++ b/src/server.c @@ -2805,6 +2805,25 @@ int prepareForShutdown(int flags) { /*================================== Commands =============================== */ +/* Sometimes Redis cannot accept write commands because there is a perstence + * error with the RDB or AOF file, and Redis is configured in order to stop + * accepting writes in such situation. This function returns if such a + * condition is active, and the type of the condition. */ +int writeCommandsDeniedByDiskError(void) { + if (server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 && + server.lastbgsave_status == C_ERR) + { + return DISK_ERROR_TYPE_AOF; + } else if (server.aof_state != AOF_OFF && + server.aof_last_write_status == C_ERR) + { + return DISK_ERROR_TYPE_RDB; + } else { + return DISK_ERROR_TYPE_NONE; + } +} + /* Return zero if strings are the same, non-zero if they are not. * The comparison is performed in a way that prevents an attacker to obtain * information about the nature of the strings just monitoring the execution diff --git a/src/server.h b/src/server.h index 21e4c4279..9af80ea32 100644 --- a/src/server.h +++ b/src/server.h @@ -1591,6 +1591,11 @@ void startLoading(FILE *fp); void loadingProgress(off_t pos); void stopLoading(void); +#define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ +#define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ +#define DISK_ERROR_TYPE_NONE 0 /* No problems, we can accept writes. */ +int writeCommandsDeniedByDiskError(void); + /* RDB persistence */ #include "rdb.h" int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi); From 0e49938b684ec3d294457391e746b42a539c88cb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:10:37 +0200 Subject: [PATCH 056/616] Better top comment for writeCommandsDeniedByDiskError(). --- src/server.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 2e0c36014..fbbfba7ff 100644 --- a/src/server.c +++ b/src/server.c @@ -2808,7 +2808,14 @@ int prepareForShutdown(int flags) { /* Sometimes Redis cannot accept write commands because there is a perstence * error with the RDB or AOF file, and Redis is configured in order to stop * accepting writes in such situation. This function returns if such a - * condition is active, and the type of the condition. */ + * condition is active, and the type of the condition. + * + * Function return values: + * + * DISK_ERROR_TYPE_NONE: No problems, we can accept writes. + * DISK_ERROR_TYPE_AOF: Don't accept writes: AOF errors. + * DISK_ERROR_TYPE_RDB: Don't accept writes: RDB errors. + */ int writeCommandsDeniedByDiskError(void) { if (server.stop_writes_on_bgsave_err && server.saveparamslen > 0 && From 11dd3f4b034566259b2f81484619975928963a89 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:14:24 +0200 Subject: [PATCH 057/616] Fix writeCommandsDeniedByDiskError() inverted return value. --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index fbbfba7ff..28750ab72 100644 --- a/src/server.c +++ b/src/server.c @@ -2821,11 +2821,11 @@ int writeCommandsDeniedByDiskError(void) { server.saveparamslen > 0 && server.lastbgsave_status == C_ERR) { - return DISK_ERROR_TYPE_AOF; + return DISK_ERROR_TYPE_RDB; } else if (server.aof_state != AOF_OFF && server.aof_last_write_status == C_ERR) { - return DISK_ERROR_TYPE_RDB; + return DISK_ERROR_TYPE_AOF; } else { return DISK_ERROR_TYPE_NONE; } From db693be00d4debc474cb52e471a9a4c91e6f263e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:16:43 +0200 Subject: [PATCH 058/616] Refactoring: replace low-level checks with writeCommandsDeniedByDiskError(). --- src/scripting.c | 10 +++------- src/server.c | 9 +++------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index b046ea1ff..2732c87fb 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -483,6 +483,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * command marked as non-deterministic was already called in the context * of this script. */ if (cmd->flags & CMD_WRITE) { + int deny_write_type = writeCommandsDeniedByDiskError(); if (server.lua_random_dirty && !server.lua_replicate_commands) { luaPushError(lua, "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); @@ -493,13 +494,8 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { { luaPushError(lua, shared.roslaveerr->ptr); goto cleanup; - } else if ((server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) || - (server.aof_state != AOF_OFF && - server.aof_last_write_status == C_ERR)) - { - if (server.aof_last_write_status == C_OK) { + } else if (deny_write_type != DISK_ERROR_TYPE_NONE) { + if (deny_write_type == DISK_ERROR_TYPE_RDB) { luaPushError(lua, shared.bgsaveerr->ptr); } else { sds aof_write_err = sdscatfmt(sdsempty(), diff --git a/src/server.c b/src/server.c index 28750ab72..66d42ee6c 100644 --- a/src/server.c +++ b/src/server.c @@ -2609,17 +2609,14 @@ int processCommand(client *c) { /* Don't accept write commands if there are problems persisting on disk * and if this is a master instance. */ - if (((server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) || - (server.aof_state != AOF_OFF && - server.aof_last_write_status == C_ERR)) && + int deny_write_type = writeCommandsDeniedByDiskError(); + if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL && (c->cmd->flags & CMD_WRITE || c->cmd->proc == pingCommand)) { flagTransaction(c); - if (server.aof_last_write_status == C_OK) + if (deny_write_type == DISK_ERROR_TYPE_RDB) addReply(c, shared.bgsaveerr); else addReplySds(c, From b0392e75ec37ffae0cb2277723168d8179861bef Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 16:41:33 +0200 Subject: [PATCH 059/616] Tranfer -> transfer typo fixed. --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index 5d48e3c43..f67073fe6 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -516,7 +516,7 @@ void spopWithCountCommand(client *c) { sdsfree(sdsele); } - /* Tranfer the old set to the client. */ + /* Transfer the old set to the client. */ setTypeIterator *si; si = setTypeInitIterator(set); while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) { From e245ed9a44583c3772445e99ffa2b14407199287 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 17:13:03 +0200 Subject: [PATCH 060/616] Cluster cron announce IP minor refactoring. --- src/cluster.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 7a6a5f782..2f3e298e0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3321,14 +3321,17 @@ void clusterCron(void) { int changed = 0; if (prev_ip == NULL && curr_ip != NULL) changed = 1; - if (prev_ip != NULL && curr_ip == NULL) changed = 1; - if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; + else if (prev_ip != NULL && curr_ip == NULL) changed = 1; + else if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; if (changed) { if (prev_ip) zfree(prev_ip); - prev_ip = curr_ip; + if (curr_ip) { + /* We always take a copy of the previous IP address, by + * duplicating the string. This way later we can check if + * the address really changed. */ prev_ip = zstrdup(prev_ip); strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN); myself->ip[NET_IP_STR_LEN-1] = '\0'; From 7a30be123794ffb339a3a73e094d255a065b6a60 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 17:28:30 +0200 Subject: [PATCH 061/616] Minor improvements to PR #5187. --- tests/integration/psync2.tcl | 10 +++++++--- tests/test_helper.tcl | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 3d9e5527a..b76f36363 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -33,9 +33,8 @@ start_server {} { set cycle 1 while {([clock seconds]-$start_time) < $duration} { - test "PSYNC2: --- CYCLE $cycle ---" { - incr cycle - } + test "PSYNC2: --- CYCLE $cycle ---" {} + incr cycle # Create a random replication layout. # Start with switching master (this simulates a failover). @@ -139,6 +138,11 @@ start_server {} { } assert {$sum == 4} } + + # Limit anyway the maximum number of cycles. This is useful when the + # test is skipped via --only option of the test suite. In that case + # we don't want to see many seconds of this test being just skipped. + if {$cycle > 50} break } test "PSYNC2: Bring the master back again for next test" { diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index ba3dce71c..5d8b149de 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -429,7 +429,7 @@ proc print_help_screen {} { "--single Just execute the specified unit (see next option). this option can be repeated." "--list-tests List all the available test units." "--only Just execute the specified test by test name. this option can be repeated." - "--skiptill Skip all units until (and including) the specified one." + "--skip-till Skip all units until (and including) the specified one." "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." @@ -458,8 +458,7 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set arg2 [lindex $argv [expr $j+2]] lappend ::global_overrides $arg lappend ::global_overrides $arg2 - incr j - incr j + incr j 2 } elseif {$opt eq {--skipfile}} { incr j set fp [open $arg r] @@ -524,6 +523,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +# If --skil-till option was given, we populate the list of single tests +# to run with everything *after* the specified unit. if {$::skip_till != ""} { set skipping 1 foreach t $::all_tests { @@ -540,6 +541,8 @@ if {$::skip_till != ""} { } } +# Override the list of tests with the specific tests we want to run +# in case there was some filter, that is --single or --skip-till options. if {[llength $::single_tests] > 0} { set ::all_tests $::single_tests } From 2f2987ffc5c3cbf8eab307782a07f20492a3d439 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 18:08:52 +0200 Subject: [PATCH 062/616] Streams IDs parsing refactoring. Related to #5184. --- src/t_stream.c | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index f233785ae..72d03b466 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1059,18 +1059,19 @@ int string2ull(const char *s, unsigned long long *value) { * to the client, otherwise C_OK is returned. The ID may be in incomplete * form, just stating the milliseconds time part of the stream. In such a case * the missing part is set according to the value of 'missing_seq' parameter. + * * The IDs "-" and "+" specify respectively the minimum and maximum IDs - * that can be represented. If 'mmid_supp' is set to 0, "-" and "+" will be + * that can be represented. If 'strict' is set to 1, "-" and "+" will be * treated as an invalid ID. * * If 'c' is set to NULL, no reply is sent to the client. */ -int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int mmid_supp) { +int streamGenericParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int strict) { char buf[128]; if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid; memcpy(buf,o->ptr,sdslen(o->ptr)+1); - if (!mmid_supp && (buf[0] == '-' || buf[0] == '+') && - buf[1] == '\0') goto invalid; + if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') + goto invalid; /* Handle the "-" and "+" special cases. */ if (buf[0] == '-' && buf[1] == '\0') { @@ -1100,6 +1101,20 @@ invalid: return C_ERR; } +/* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to + * 0, to be used when - and + are accetable IDs. */ +int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { + return streamGenericParseIDOrReply(c,o,id,missing_seq,0); +} + +/* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to + * 1, to be used when we want to return an error if the special IDs + or - + * are provided. */ +int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { + return streamGenericParseIDOrReply(c,o,id,missing_seq,1); +} + + /* XADD key [MAXLEN ] [field value] [field value] ... */ void xaddCommand(client *c) { streamID id; @@ -1137,7 +1152,7 @@ void xaddCommand(client *c) { maxlen_arg_idx = i; } else { /* If we are here is a syntax error or a valid ID. */ - if (streamParseIDOrReply(c,c->argv[i],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[i],&id,0) != C_OK) return; id_given = 1; break; } @@ -1206,8 +1221,8 @@ void xrangeGenericCommand(client *c, int rev) { robj *startarg = rev ? c->argv[3] : c->argv[2]; robj *endarg = rev ? c->argv[2] : c->argv[3]; - if (streamParseIDOrReply(c,startarg,&startid,0,1) == C_ERR) return; - if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX,1) == C_ERR) return; + if (streamParseIDOrReply(c,startarg,&startid,0) == C_ERR) return; + if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX) == C_ERR) return; /* Parse the COUNT option if any. */ if (c->argc > 4) { @@ -1393,7 +1408,7 @@ void xreadCommand(client *c) { ids[id_idx].seq = UINT64_MAX; continue; } - if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0,0) != C_OK) + if (streamParseStrictIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK) goto cleanup; } @@ -1667,7 +1682,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0,0) != C_OK) { + } else if (streamParseStrictIDOrReply(c,c->argv[4],&id,0) != C_OK) { return; } streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); @@ -1684,7 +1699,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0,1) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { return; } cg->last_id = id; @@ -1744,7 +1759,7 @@ void xackCommand(client *c) { for (int j = 3; j < c->argc; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL: it will have a reference to the @@ -1791,9 +1806,9 @@ void xpendingCommand(client *c) { if (c->argc >= 6) { if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[3],&startid,0,1) == C_ERR) + if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX,1) == C_ERR) + if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR) return; } @@ -2002,7 +2017,7 @@ void xclaimCommand(client *c) { int j; for (j = 4; j < c->argc; j++) { streamID id; - if (streamParseIDOrReply(NULL,c->argv[j],&id,0,0) != C_OK) break; + if (streamParseStrictIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; } int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */ @@ -2061,7 +2076,7 @@ void xclaimCommand(client *c) { for (int j = 5; j <= last_id_arg; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL. */ @@ -2144,13 +2159,13 @@ void xdelCommand(client *c) { * executed because at some point an invalid ID is parsed. */ streamID id; for (int j = 2; j < c->argc; j++) { - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; } /* Actually apply the command. */ int deleted = 0; for (int j = 2; j < c->argc; j++) { - streamParseIDOrReply(c,c->argv[j],&id,0,0); /* Retval already checked. */ + streamParseStrictIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */ deleted += streamDeleteItem(s,&id); } From 20c6a7fe2c134ad21bfc4ce50e548c3a055e93d0 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 17 Jul 2018 23:57:42 +0800 Subject: [PATCH 063/616] Streams: propagate original MAXLEN argument in XADD context If we rewrite the MAXLEN argument as zero when no trimming was performed, date between master and slave and aof will be inconsistent, because `xtrim maxlen 0` means delete all entries in stream. --- src/t_stream.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 72d03b466..5814aa132 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1186,18 +1186,9 @@ void xaddCommand(client *c) { notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); server.dirty++; - /* Remove older elements if MAXLEN was specified. */ - if (maxlen >= 0) { - if (!streamTrimByLength(s,maxlen,approx_maxlen)) { - /* If no trimming was performed, for instance because approximated - * trimming length was specified, rewrite the MAXLEN argument - * as zero, so that the command is propagated without trimming. */ - robj *zeroobj = createStringObjectFromLongLong(0); - rewriteClientCommandArgument(c,maxlen_arg_idx,zeroobj); - decrRefCount(zeroobj); - } else { - notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); - } + /* Notify xtrim event if needed. */ + if (maxlen >= 0 && streamTrimByLength(s,maxlen,approx_maxlen)) { + notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); } /* Let's rewrite the ID argument with the one actually generated for From da6b7516f187b85e7f9a8a2390f02e9c1dc2c7aa Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 00:12:24 +0800 Subject: [PATCH 064/616] Streams: XTRIM will return an error if MAXLEN with a count < 0 --- src/t_stream.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 5814aa132..bab5b74ff 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2192,7 +2192,7 @@ void xtrimCommand(client *c) { /* Argument parsing. */ int trim_strategy = TRIM_STRATEGY_NONE; - long long maxlen = 0; /* 0 means no maximum length. */ + long long maxlen = -1; /* If left to -1 no trimming is performed. */ int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so the maxium length is not applied verbatim. */ @@ -2211,6 +2211,11 @@ void xtrimCommand(client *c) { } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; + + if (maxlen < 0) { + addReplyError(c,"The MAXLEN argument must be >= 0."); + return; + } i++; } else { addReply(c,shared.syntaxerr); From 14d6318b3225c010f28d26c5563c8140c86c1292 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 00:24:50 +0800 Subject: [PATCH 065/616] Streams: reset approx_maxlen in every maxlen loop --- src/t_stream.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index bab5b74ff..7533ba7b0 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1135,6 +1135,7 @@ void xaddCommand(client *c) { * creation. */ break; } else if (!strcasecmp(opt,"maxlen") && moreargs) { + approx_maxlen = 0; char *next = c->argv[i+1]->ptr; /* Check for the form MAXLEN ~ . */ if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { @@ -2202,6 +2203,7 @@ void xtrimCommand(client *c) { int moreargs = (c->argc-1) - i; /* Number of additional arguments. */ char *opt = c->argv[i]->ptr; if (!strcasecmp(opt,"maxlen") && moreargs) { + approx_maxlen = 0; trim_strategy = TRIM_STRATEGY_MAXLEN; char *next = c->argv[i+1]->ptr; /* Check for the form MAXLEN ~ . */ From 9042d1c24966bf229b3fa8d94ada42ebebed7adf Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 01:58:14 +0800 Subject: [PATCH 066/616] Streams: propagate specified MAXLEN instead of approximated Slaves and rebooting redis may have different radix tree struct, by different stream* config options. So propagating approximated MAXLEN to AOF/slaves may lead to date inconsistency. --- src/t_stream.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 7533ba7b0..7eaf0c547 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1114,8 +1114,7 @@ int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missin return streamGenericParseIDOrReply(c,o,id,missing_seq,1); } - -/* XADD key [MAXLEN ] [field value] [field value] ... */ +/* XADD key [MAXLEN [~|=] ] [field value] [field value] ... */ void xaddCommand(client *c) { streamID id; int id_given = 0; /* Was an ID different than "*" specified? */ @@ -1141,6 +1140,8 @@ void xaddCommand(client *c) { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { approx_maxlen = 1; i++; + } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') { + i++; } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; @@ -1187,9 +1188,22 @@ void xaddCommand(client *c) { notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); server.dirty++; - /* Notify xtrim event if needed. */ - if (maxlen >= 0 && streamTrimByLength(s,maxlen,approx_maxlen)) { - notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); + if (maxlen >= 0) { + /* Notify xtrim event if needed. */ + if (streamTrimByLength(s,maxlen,approx_maxlen)) { + notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); + } + + /* Rewrite approximated MAXLEN as specified s->length. */ + if (approx_maxlen) { + robj *maxlen_obj = createStringObjectFromLongLong(s->length); + rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj); + decrRefCount(maxlen_obj); + + robj *equal_obj = createStringObject("=",1); + rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj); + decrRefCount(equal_obj); + } } /* Let's rewrite the ID argument with the one actually generated for @@ -2174,7 +2188,7 @@ void xdelCommand(client *c) { * * List of options: * - * MAXLEN [~] -- Trim so that the stream will be capped at + * MAXLEN [~|=] -- Trim so that the stream will be capped at * the specified length. Use ~ before the * count in order to demand approximated trimming * (like XADD MAXLEN option). @@ -2196,6 +2210,7 @@ void xtrimCommand(client *c) { long long maxlen = -1; /* If left to -1 no trimming is performed. */ int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so the maxium length is not applied verbatim. */ + int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */ /* Parse options. */ int i = 2; /* Start of options. */ @@ -2210,6 +2225,8 @@ void xtrimCommand(client *c) { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { approx_maxlen = 1; i++; + } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') { + i++; } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; @@ -2219,6 +2236,7 @@ void xtrimCommand(client *c) { return; } i++; + maxlen_arg_idx = i; } else { addReply(c,shared.syntaxerr); return; @@ -2239,6 +2257,17 @@ void xtrimCommand(client *c) { signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); server.dirty += deleted; + + /* Rewrite approximated MAXLEN as specified s->length. */ + if (approx_maxlen) { + robj *maxlen_obj = createStringObjectFromLongLong(s->length); + rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj); + decrRefCount(maxlen_obj); + + robj *equal_obj = createStringObject("=",1); + rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj); + decrRefCount(equal_obj); + } } addReplyLongLong(c,deleted); } From 60acac4cd02913385c461465d4cca06d6c015ba7 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 16:55:25 +0800 Subject: [PATCH 067/616] Streams: add test cases for XADD/XTRIM maxlen --- tests/unit/type/stream.tcl | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 5cf6805d7..2b69a2e9e 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -317,3 +317,49 @@ start_server { assert_equal [r xrevrange teststream2 1234567891245 -] {{1234567891240-0 {key1 value2}} {1234567891230-0 {key1 value1}}} } } + +start_server {tags {"stream"} overrides {appendonly yes}} { + test {XADD with MAXLEN > xlen can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XADD mystream MAXLEN 200 * xitem v + incr j + assert {[r xlen mystream] == $j} + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == $j} + } +} + +start_server {tags {"stream"} overrides {appendonly yes}} { + test {XADD with ~ MAXLEN can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XADD mystream MAXLEN ~ $j * xitem v + incr j + assert {[r xlen mystream] == $j} + r config set stream-node-max-entries 1 + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == $j} + } +} + +start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} { + test {XTRIM with ~ MAXLEN can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XTRIM mystream MAXLEN ~ 85 + assert {[r xlen mystream] == 89} + r config set stream-node-max-entries 1 + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == 90} + } +} From 6dd0d6f7dd1fc486ede902c43d299640ce08cc2c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:27:56 +0200 Subject: [PATCH 068/616] zsetAdd() refactored adding zslUpdateScore(). --- src/t_zset.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 92abb49d5..06fb61063 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -244,6 +244,23 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { return 0; /* not found */ } +/* Update the score of an elmenent inside the sorted set skiplist. + * Note that the element must exist and must match 'score'. + * This function does not update the score in the hash table side, the + * caller should take care of it. + * + * The function returns the updated element skiplist node pointer. */ +zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { + zskiplistNode *node, *newnode; + serverAssert(zslDelete(zsl,curscore,ele,&node)); + newnode = zslInsert(zsl,newscore,node->ele); + /* We reused the node->ele SDS string, free the node now + * since zslInsert created a new one. */ + node->ele = NULL; + zslFreeNode(node); + return newnode; +} + int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); } @@ -1341,13 +1358,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { /* Remove and re-insert when score changes. */ if (score != curscore) { - zskiplistNode *node; - serverAssert(zslDelete(zs->zsl,curscore,ele,&node)); - znode = zslInsert(zs->zsl,score,node->ele); - /* We reused the node->ele SDS string, free the node now - * since zslInsert created a new one. */ - node->ele = NULL; - zslFreeNode(node); + znode = zslUpdateScore(zs->zsl,curscore,ele,score); /* Note that we did not removed the original element from * the hash table representing the sorted set, so we just * update the score. */ From 0b800c4332681400b1a0c59068b3b542da23b381 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:50:31 +0200 Subject: [PATCH 069/616] Optimize zslUpdateScore() as asked in #5179. --- src/t_zset.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 06fb61063..33e9aa141 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -250,6 +250,49 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { * caller should take care of it. * * The function returns the updated element skiplist node pointer. */ +zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { + zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; + int i; + + x = zsl->header; + for (i = zsl->level-1; i >= 0; i--) { + while (x->level[i].forward && + (x->level[i].forward->score < curscore || + (x->level[i].forward->score == curscore && + sdscmp(x->level[i].forward->ele,ele) < 0))) + { + x = x->level[i].forward; + } + update[i] = x; + } + + /* Jump to our element: note that this function assumes that the + * element with the matching score exists. */ + x = x->level[0].forward; + serverAssert(x && curscore == x->score && sdscmp(x->ele,ele) == 0); + + /* If the node, after the score update, would be still exactly + * at the same position, we can just update the score without + * actually removing and re-inserting the element in the skiplist. */ + if ((x->backward == NULL || x->backward->score <= newscore) && + (x->level[0].forward == NULL || x->level[0].forward->score >= newscore)) + { + x->score = newscore; + return x; + } + + /* No way to reuse the old node: we need to remove and insert a new + * one at a different place. */ + zslDeleteNode(zsl, x, update); + zskiplistNode *newnode = zslInsert(zsl,newscore,x->ele); + /* We reused the old node x->ele SDS string, free the node now + * since zslInsert created a new one. */ + x->ele = NULL; + zslFreeNode(x); + return newnode; +} + +#if 0 zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { zskiplistNode *node, *newnode; serverAssert(zslDelete(zsl,curscore,ele,&node)); @@ -260,6 +303,7 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n zslFreeNode(node); return newnode; } +#endif int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); From e3e94ec35cce5acf80d429241b4bbc3fedfa8276 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:50:49 +0200 Subject: [PATCH 070/616] Remove old commented zslUpdateScore() from source. --- src/t_zset.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 33e9aa141..91fe01d76 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -292,19 +292,6 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n return newnode; } -#if 0 -zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { - zskiplistNode *node, *newnode; - serverAssert(zslDelete(zsl,curscore,ele,&node)); - newnode = zslInsert(zsl,newscore,node->ele); - /* We reused the node->ele SDS string, free the node now - * since zslInsert created a new one. */ - node->ele = NULL; - zslFreeNode(node); - return newnode; -} -#endif - int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); } From 12ff0c0d79ce673a5e83c8cbff1cdbf773baf800 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:53:06 +0200 Subject: [PATCH 071/616] Explain what's the point of zslUpdateScore() in top comment. --- src/t_zset.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 91fe01d76..16ec715e8 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -249,6 +249,11 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { * This function does not update the score in the hash table side, the * caller should take care of it. * + * Note that this function attempts to just update the node, in case after + * the score update, the node would be exactly at the same position. + * Otherwise the skiplist is modified by removing and re-adding a new + * element, which is more costly. + * * The function returns the updated element skiplist node pointer. */ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; From 29226a3919ef1f74bc161b31972193ce30bc0ed6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:54:15 +0200 Subject: [PATCH 072/616] More commenting of zslUpdateScore(). --- src/t_zset.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 16ec715e8..a794b44c3 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -259,6 +259,8 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; int i; + /* We need to seek to element to update to start: this is useful anyway, + * we'll have to update or remove it. */ x = zsl->header; for (i = zsl->level-1; i >= 0; i--) { while (x->level[i].forward && From 2f282aee0b8e500697b627254b170f21d93dd4a8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 19:01:40 +0200 Subject: [PATCH 073/616] Fix zslUpdateScore() edge case. When the element new score is the same of prev/next node, the lexicographical order kicks in, so we can safely update the node in place only when the new score is strictly between the adjacent nodes but never equal to one of them. Technically speaking we could do extra checks to make sure that even if the score is the same as one of the adjacent nodes, we can still update on place, but this rarely happens, so probably not a good deal to make it more complex. Related to #5179. --- src/t_zset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index a794b44c3..db381b592 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -281,8 +281,8 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n /* If the node, after the score update, would be still exactly * at the same position, we can just update the score without * actually removing and re-inserting the element in the skiplist. */ - if ((x->backward == NULL || x->backward->score <= newscore) && - (x->level[0].forward == NULL || x->level[0].forward->score >= newscore)) + if ((x->backward == NULL || x->backward->score < newscore) && + (x->level[0].forward == NULL || x->level[0].forward->score > newscore)) { x->score = newscore; return x; From d506334b678ea1f0ad2f28b1144365ac7751999a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 14:14:39 +0200 Subject: [PATCH 074/616] Test: new sorted set skiplist order consistency. This should be able to find new bugs and regressions about the new sorted set update function when ZADD is used to update an element already existing. The test is able to find the bug fixed at 2f282aee immediately. --- tests/unit/type/zset.tcl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index bebba96d9..cf54ae839 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1185,4 +1185,30 @@ start_server {tags {"zset"}} { stressers ziplist stressers skiplist } + + test {ZSET skiplist order consistency when elements are moved} { + set original_max [lindex [r config get zset-max-ziplist-entries] 1] + r config set zset-max-ziplist-entries 0 + for {set times 0} {$times < 10} {incr times} { + r del zset + for {set j 0} {$j < 1000} {incr j} { + r zadd zset [randomInt 50] ele-[randomInt 10] + } + + # Make sure that element ordering is correct + set prev_element {} + set prev_score -1 + foreach {element score} [r zrange zset 0 -1 WITHSCORES] { + # Assert that elements are in increasing ordering + assert { + $prev_score < $score || + ($prev_score == $score && + [string compare $prev_element $element] == -1) + } + set prev_element $element + set prev_score $score + } + } + r config set zset-max-ziplist-entries $original_max + } } From 26897c03e2b9a15c5a574c41763aaae7b9c7d66e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 18:49:49 +0200 Subject: [PATCH 075/616] Test suite: new --stop option. It pauses the test execution once the first failure is found. --- tests/test_helper.tcl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 5d8b149de..753a0f9b9 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -89,6 +89,7 @@ set ::last_progress [clock seconds] set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 set ::wait_server 0 +set ::stop_on_failure 0 # Set to 1 when we are running in client mode. The Redis test uses a # server-client model to run tests simultaneously. The server instance @@ -302,6 +303,11 @@ proc read_from_test_client fd { puts $err lappend ::failed_tests $err set ::active_clients_task($fd) "(ERR) $data" + if {$::stop_on_failure} { + puts -nonewline "(Test stopped, press enter to continue)" + flush stdout + gets stdin + } } elseif {$status eq {exception}} { puts "\[[colorstr red $status]\]: $data" kill_clients @@ -433,10 +439,11 @@ proc print_help_screen {} { "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." - "--config extra config file argument" - "--skipfile name of a file containing test names that should be skipped (one per line)" - "--dont-clean don't delete redis log files after the run" - "--wait-server wait after server is started (so that you can attach a debugger)" + "--config Extra config file argument" + "--skipfile Name of a file containing test names that should be skipped (one per line)" + "--dont-clean Don't delete redis log files after the run" + "--stop Blocks once the first test fails" + "--wait-server Wait after server is started (so that you can attach a debugger)" "--help Print this help screen." } "\n"] } @@ -511,6 +518,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set ::dont_clean 1 } elseif {$opt eq {--wait-server}} { set ::wait_server 1 + } elseif {$opt eq {--stop}} { + set ::stop_on_failure 1 } elseif {$opt eq {--timeout}} { set ::timeout $arg incr j From 3d56311f0ca077fc4661e132508ceca30e74de74 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 19:07:17 +0200 Subject: [PATCH 076/616] Test suite: add --loop option. Very useful with --stop in order to catch heisenbugs. --- tests/test_helper.tcl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 753a0f9b9..1a986e2f7 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -90,6 +90,7 @@ set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 set ::wait_server 0 set ::stop_on_failure 0 +set ::loop 0 # Set to 1 when we are running in client mode. The Redis test uses a # server-client model to run tests simultaneously. The server instance @@ -370,6 +371,9 @@ proc signal_idle_client fd { send_data_packet $fd run [lindex $::all_tests $::next_test] lappend ::active_clients $fd incr ::next_test + if {$::loop && $::next_test == [llength $::all_tests]} { + set ::next_test 0 + } } else { lappend ::idle_clients $fd if {[llength $::active_clients] == 0} { @@ -439,11 +443,12 @@ proc print_help_screen {} { "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." - "--config Extra config file argument" - "--skipfile Name of a file containing test names that should be skipped (one per line)" - "--dont-clean Don't delete redis log files after the run" - "--stop Blocks once the first test fails" - "--wait-server Wait after server is started (so that you can attach a debugger)" + "--config Extra config file argument." + "--skipfile Name of a file containing test names that should be skipped (one per line)." + "--dont-clean Don't delete redis log files after the run." + "--stop Blocks once the first test fails." + "--loop Execute the specified set of tests forever." + "--wait-server Wait after server is started (so that you can attach a debugger)." "--help Print this help screen." } "\n"] } @@ -520,6 +525,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set ::wait_server 1 } elseif {$opt eq {--stop}} { set ::stop_on_failure 1 + } elseif {$opt eq {--loop}} { + set ::loop 1 } elseif {$opt eq {--timeout}} { set ::timeout $arg incr j From 0ce8323c0d6908edd126c011ad2d44a72a022c48 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 3 Aug 2018 12:46:04 +0200 Subject: [PATCH 077/616] Fix AOF comment to report the current behavior. Realted to #5201. --- src/aof.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index be416ec4e..f8f26bdfe 100644 --- a/src/aof.c +++ b/src/aof.c @@ -798,7 +798,9 @@ int loadAppendOnlyFile(char *filename) { } /* This point can only be reached when EOF is reached without errors. - * If the client is in the middle of a MULTI/EXEC, log error and quit. */ + * If the client is in the middle of a MULTI/EXEC, handle it as it was + * a short read, even if technically the protocol is correct: we want + * to remove the unprocessed tail and continue. */ if (fakeClient->flags & CLIENT_MULTI) goto uxeof; loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ From fe56c6740520b6fef2646d757896861c49faee0a Mon Sep 17 00:00:00 2001 From: shenlongxing Date: Fri, 3 Aug 2018 19:01:15 +0800 Subject: [PATCH 078/616] Fix stream command paras --- src/server.c | 10 +++++----- src/t_stream.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server.c b/src/server.c index 66d42ee6c..b537ee04a 100644 --- a/src/server.c +++ b/src/server.c @@ -310,14 +310,14 @@ struct redisCommand redisCommandTable[] = { {"xrange",xrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xrevrange",xrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xlen",xlenCommand,2,"rF",0,NULL,1,1,1,0,0}, - {"xread",xreadCommand,-3,"rs",0,xreadGetKeys,1,1,1,0,0}, - {"xreadgroup",xreadCommand,-3,"ws",0,xreadGetKeys,1,1,1,0,0}, + {"xread",xreadCommand,-4,"rs",0,xreadGetKeys,1,1,1,0,0}, + {"xreadgroup",xreadCommand,-7,"ws",0,xreadGetKeys,1,1,1,0,0}, {"xgroup",xgroupCommand,-2,"wm",0,NULL,2,2,1,0,0}, - {"xack",xackCommand,-3,"wF",0,NULL,1,1,1,0,0}, + {"xack",xackCommand,-4,"wF",0,NULL,1,1,1,0,0}, {"xpending",xpendingCommand,-3,"r",0,NULL,1,1,1,0,0}, - {"xclaim",xclaimCommand,-5,"wF",0,NULL,1,1,1,0,0}, + {"xclaim",xclaimCommand,-6,"wF",0,NULL,1,1,1,0,0}, {"xinfo",xinfoCommand,-2,"r",0,NULL,2,2,1,0,0}, - {"xdel",xdelCommand,-2,"wF",0,NULL,1,1,1,0,0}, + {"xdel",xdelCommand,-3,"wF",0,NULL,1,1,1,0,0}, {"xtrim",xtrimCommand,-2,"wF",0,NULL,1,1,1,0,0}, {"post",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"host:",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, diff --git a/src/t_stream.c b/src/t_stream.c index 72d03b466..d31563217 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2040,12 +2040,12 @@ void xclaimCommand(client *c) { } else if (!strcasecmp(opt,"TIME") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime, - "Invalid IDLE option argument for XCLAIM") + "Invalid TIME option argument for XCLAIM") != C_OK) return; } else if (!strcasecmp(opt,"RETRYCOUNT") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&retrycount, - "Invalid IDLE option argument for XCLAIM") + "Invalid RETRYCOUNT option argument for XCLAIM") != C_OK) return; } else { addReplyErrorFormat(c,"Unrecognized XCLAIM option '%s'",opt); From eb87da6127c605be06073903a0bbaff792b90d70 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 2 Aug 2018 14:59:28 +0800 Subject: [PATCH 079/616] AOF: discard if we lost EXEC when loading aof --- src/aof.c | 15 +++++++++++++-- src/multi.c | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/aof.c b/src/aof.c index f8f26bdfe..0c7f28269 100644 --- a/src/aof.c +++ b/src/aof.c @@ -677,6 +677,7 @@ int loadAppendOnlyFile(char *filename) { int old_aof_state = server.aof_state; long loops = 0; off_t valid_up_to = 0; /* Offset of latest well-formed command loaded. */ + off_t valid_before_multi = 0; /* Offset before MULTI command loaded. */ if (fp == NULL) { serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno)); @@ -781,9 +782,15 @@ int loadAppendOnlyFile(char *filename) { exit(1); } + if (cmd == server.multiCommand) valid_before_multi = valid_up_to; + /* Run the command in the context of a fake client */ fakeClient->cmd = cmd; - cmd->proc(fakeClient); + if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { + queueMultiCommand(fakeClient); + } else { + cmd->proc(fakeClient); + } /* The fake client should not have a reply */ serverAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0); @@ -801,7 +808,11 @@ int loadAppendOnlyFile(char *filename) { * If the client is in the middle of a MULTI/EXEC, handle it as it was * a short read, even if technically the protocol is correct: we want * to remove the unprocessed tail and continue. */ - if (fakeClient->flags & CLIENT_MULTI) goto uxeof; + if (fakeClient->flags & CLIENT_MULTI) { + serverLog(LL_WARNING,"!!! Warning: we lost EXEC in the middle of transaction, discard !!!"); + valid_up_to = valid_before_multi; + goto uxeof; + } loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ fclose(fp); diff --git a/src/multi.c b/src/multi.c index 112ce0605..8159adcb3 100644 --- a/src/multi.c +++ b/src/multi.c @@ -158,7 +158,7 @@ void execCommand(client *c) { must_propagate = 1; } - call(c,CMD_CALL_FULL); + call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; From a3a146052509b2d3a58e36d3511ef238153b6ac3 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sat, 4 Aug 2018 01:06:53 +0800 Subject: [PATCH 080/616] Streams: update listpack with new pointer in XDEL --- src/t_stream.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 77fbf4645..c5d3c7f9d 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -724,6 +724,9 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { p = lpNext(lp,p); /* Seek deleted field. */ aux = lpGetInteger(p); lp = lpReplaceInteger(lp,&p,aux+1); + + /* Update the listpack with the new pointer. */ + raxInsert(si->stream->rax,si->ri.key,si->ri.key_len,lp,NULL); } /* Update the number of entries counter. */ From bd01334da1d27367ad9887f159f2860f7e8ba5ac Mon Sep 17 00:00:00 2001 From: Jeffrey Lovitz Date: Sun, 12 Aug 2018 12:47:01 -0400 Subject: [PATCH 081/616] CLI Help text loop verifies arg count --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0e8777bd2..247d1f734 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -545,7 +545,7 @@ static void cliIntegrateHelp(void) { ch->params = sdscat(ch->params,"key "); args--; } - while(args--) ch->params = sdscat(ch->params,"arg "); + while(args-- > 0) ch->params = sdscat(ch->params,"arg "); if (entry->element[1]->integer < 0) ch->params = sdscat(ch->params,"...options..."); ch->summary = "Help not available"; From f4b27ae22272b9a50e970aa308ba57c3c3dbd985 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 13 Aug 2018 17:36:54 +0300 Subject: [PATCH 082/616] script cache memory in INFO and MEMORY includes both script code and overheads --- src/object.c | 2 +- src/server.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object.c b/src/object.c index 406816140..90ccbc916 100644 --- a/src/object.c +++ b/src/object.c @@ -1017,7 +1017,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; - mem = 0; + mem = server.lua_scripts_mem; mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + dictSlots(server.lua_scripts) * sizeof(dictEntry*); mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + diff --git a/src/server.c b/src/server.c index b537ee04a..726147bf1 100644 --- a/src/server.c +++ b/src/server.c @@ -3186,7 +3186,7 @@ sds genRedisInfoString(char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); - bytesToHuman(used_memory_scripts_hmem,server.lua_scripts_mem); + bytesToHuman(used_memory_scripts_hmem,mh->lua_caches); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -3251,7 +3251,7 @@ sds genRedisInfoString(char *section) { total_system_hmem, memory_lua, used_memory_lua_hmem, - server.lua_scripts_mem, + mh->lua_caches, used_memory_scripts_hmem, dictSize(server.lua_scripts), server.maxmemory, From 14c4ddb5a62a52e30ee169d36b516a78a410a5b4 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 00:43:36 +0800 Subject: [PATCH 083/616] pipeline: do not sdsrange querybuf unless all commands processed This is an optimization for processing pipeline, we discussed a problem in issue #5229: clients may be paused if we apply `CLIENT PAUSE` command, and then querybuf may grow too large, the cost of memmove in sdsrange after parsing a completed command will be horrible. The optimization is that parsing all commands in queyrbuf , after that we can just call sdsrange only once. --- src/networking.c | 87 ++++++++++++++++++++++++++---------------------- src/server.h | 1 + 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/networking.c b/src/networking.c index af7422178..945fdd961 100644 --- a/src/networking.c +++ b/src/networking.c @@ -110,6 +110,7 @@ client *createClient(int fd) { c->fd = fd; c->name = NULL; c->bufpos = 0; + c->qb_pos = 0; c->querybuf = sdsempty(); c->pending_querybuf = sdsempty(); c->querybuf_peak = 0; @@ -1119,11 +1120,11 @@ int processInlineBuffer(client *c) { size_t querylen; /* Search for end of line */ - newline = strchr(c->querybuf,'\n'); + newline = strchr(c->querybuf+c->qb_pos,'\n'); /* Nothing to do without a \r\n */ if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big inline request"); setProtocolError("too big inline request",c,0); } @@ -1131,12 +1132,12 @@ int processInlineBuffer(client *c) { } /* Handle the \r\n case. */ - if (newline && newline != c->querybuf && *(newline-1) == '\r') + if (newline && newline != c->querybuf+c->qb_pos && *(newline-1) == '\r') newline--, linefeed_chars++; /* Split the input buffer up to the \r\n */ - querylen = newline-(c->querybuf); - aux = sdsnewlen(c->querybuf,querylen); + querylen = newline-(c->querybuf+c->qb_pos); + aux = sdsnewlen(c->querybuf+c->qb_pos,querylen); argv = sdssplitargs(aux,&argc); sdsfree(aux); if (argv == NULL) { @@ -1152,7 +1153,8 @@ int processInlineBuffer(client *c) { c->repl_ack_time = server.unixtime; /* Leave data after the first line of the query in the buffer */ - sdsrange(c->querybuf,querylen+linefeed_chars,-1); + sdsrange(c->querybuf,c->qb_pos+querylen+linefeed_chars,-1); + c->qb_pos = 0; /* Setup argv array on client structure */ if (argc) { @@ -1182,10 +1184,10 @@ static void setProtocolError(const char *errstr, client *c, long pos) { /* Sample some protocol to given an idea about what was inside. */ char buf[256]; - if (sdslen(c->querybuf) < PROTO_DUMP_LEN) { - snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%s'", c->querybuf); + if (sdslen(c->querybuf)-c->qb_pos < PROTO_DUMP_LEN) { + snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%s'", c->querybuf+c->qb_pos); } else { - snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%.*s' (... more %zu bytes ...) '%.*s'", PROTO_DUMP_LEN/2, c->querybuf, sdslen(c->querybuf)-PROTO_DUMP_LEN, PROTO_DUMP_LEN/2, c->querybuf+sdslen(c->querybuf)-PROTO_DUMP_LEN/2); + snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%.*s' (... more %zu bytes ...) '%.*s'", PROTO_DUMP_LEN/2, c->querybuf+c->qb_pos, sdslen(c->querybuf)-c->qb_pos-PROTO_DUMP_LEN, PROTO_DUMP_LEN/2, c->querybuf+sdslen(c->querybuf)-PROTO_DUMP_LEN/2); } /* Remove non printable chars. */ @@ -1202,6 +1204,7 @@ static void setProtocolError(const char *errstr, client *c, long pos) { } c->flags |= CLIENT_CLOSE_AFTER_REPLY; sdsrange(c->querybuf,pos,-1); + c->qb_pos -= pos; } /* Process the query buffer for client 'c', setting up the client argument @@ -1217,7 +1220,6 @@ static void setProtocolError(const char *errstr, client *c, long pos) { * to be '*'. Otherwise for inline commands processInlineBuffer() is called. */ int processMultibulkBuffer(client *c) { char *newline = NULL; - long pos = 0; int ok; long long ll; @@ -1226,9 +1228,9 @@ int processMultibulkBuffer(client *c) { serverAssertWithInfo(c,NULL,c->argc == 0); /* Multi bulk length cannot be read without a \r\n */ - newline = strchr(c->querybuf,'\r'); + newline = strchr(c->querybuf+c->qb_pos,'\r'); if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big mbulk count string"); setProtocolError("too big mbulk count string",c,0); } @@ -1236,22 +1238,23 @@ int processMultibulkBuffer(client *c) { } /* Buffer should also contain \n */ - if (newline-(c->querybuf) > ((signed)sdslen(c->querybuf)-2)) + if (newline-(c->querybuf+c->qb_pos) > (ssize_t)(sdslen(c->querybuf)-c->qb_pos-2)) return C_ERR; /* We know for sure there is a whole line since newline != NULL, * so go ahead and find out the multi bulk length. */ - serverAssertWithInfo(c,NULL,c->querybuf[0] == '*'); - ok = string2ll(c->querybuf+1,newline-(c->querybuf+1),&ll); + serverAssertWithInfo(c,NULL,c->querybuf[c->qb_pos] == '*'); + ok = string2ll(c->querybuf+1+c->qb_pos,newline-(c->querybuf+1+c->qb_pos),&ll); if (!ok || ll > 1024*1024) { addReplyError(c,"Protocol error: invalid multibulk length"); - setProtocolError("invalid mbulk count",c,pos); + setProtocolError("invalid mbulk count",c,c->qb_pos); return C_ERR; } - pos = (newline-c->querybuf)+2; + c->qb_pos = (newline-c->querybuf)+2; if (ll <= 0) { - sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; return C_OK; } @@ -1266,9 +1269,9 @@ int processMultibulkBuffer(client *c) { while(c->multibulklen) { /* Read bulk length if unknown */ if (c->bulklen == -1) { - newline = strchr(c->querybuf+pos,'\r'); + newline = strchr(c->querybuf+c->qb_pos,'\r'); if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c, "Protocol error: too big bulk count string"); setProtocolError("too big bulk count string",c,0); @@ -1278,25 +1281,25 @@ int processMultibulkBuffer(client *c) { } /* Buffer should also contain \n */ - if (newline-(c->querybuf) > ((signed)sdslen(c->querybuf)-2)) + if (newline-(c->querybuf+c->qb_pos) > (ssize_t)(sdslen(c->querybuf)-c->qb_pos-2)) break; - if (c->querybuf[pos] != '$') { + if (c->querybuf[c->qb_pos] != '$') { addReplyErrorFormat(c, "Protocol error: expected '$', got '%c'", - c->querybuf[pos]); - setProtocolError("expected $ but got something else",c,pos); + c->querybuf[c->qb_pos]); + setProtocolError("expected $ but got something else",c,c->qb_pos); return C_ERR; } - ok = string2ll(c->querybuf+pos+1,newline-(c->querybuf+pos+1),&ll); + ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll); if (!ok || ll < 0 || ll > server.proto_max_bulk_len) { addReplyError(c,"Protocol error: invalid bulk length"); - setProtocolError("invalid bulk length",c,pos); + setProtocolError("invalid bulk length",c,c->qb_pos); return C_ERR; } - pos += newline-(c->querybuf+pos)+2; + c->qb_pos = newline-c->querybuf+2; if (ll >= PROTO_MBULK_BIG_ARG) { size_t qblen; @@ -1304,8 +1307,8 @@ int processMultibulkBuffer(client *c) { * try to make it likely that it will start at c->querybuf * boundary so that we can optimize object creation * avoiding a large copy of data. */ - sdsrange(c->querybuf,pos,-1); - pos = 0; + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; qblen = sdslen(c->querybuf); /* Hint the sds library about the amount of bytes this string is * going to contain. */ @@ -1316,14 +1319,14 @@ int processMultibulkBuffer(client *c) { } /* Read bulk argument */ - if (sdslen(c->querybuf)-pos < (size_t)(c->bulklen+2)) { + if (sdslen(c->querybuf)-c->qb_pos < (size_t)(c->bulklen+2)) { /* Not enough data (+2 == trailing \r\n) */ break; } else { /* Optimization: if the buffer contains JUST our bulk element * instead of creating a new object by *copying* the sds we * just use the current sds string. */ - if (pos == 0 && + if (c->qb_pos == 0 && c->bulklen >= PROTO_MBULK_BIG_ARG && sdslen(c->querybuf) == (size_t)(c->bulklen+2)) { @@ -1333,20 +1336,16 @@ int processMultibulkBuffer(client *c) { * likely... */ c->querybuf = sdsnewlen(SDS_NOINIT,c->bulklen+2); sdsclear(c->querybuf); - pos = 0; } else { c->argv[c->argc++] = - createStringObject(c->querybuf+pos,c->bulklen); - pos += c->bulklen+2; + createStringObject(c->querybuf+c->qb_pos,c->bulklen); + c->qb_pos += c->bulklen+2; } c->bulklen = -1; c->multibulklen--; } } - /* Trim to pos */ - if (pos) sdsrange(c->querybuf,pos,-1); - /* We're done when c->multibulk == 0 */ if (c->multibulklen == 0) return C_OK; @@ -1360,8 +1359,9 @@ int processMultibulkBuffer(client *c) { * pending query buffer, already representing a full command, to process. */ void processInputBuffer(client *c) { server.current_client = c; + /* Keep processing while there is something in the input buffer */ - while(sdslen(c->querybuf)) { + while(c->qb_pos < sdslen(c->querybuf)) { /* Return if clients are paused. */ if (!(c->flags & CLIENT_SLAVE) && clientsArePaused()) break; @@ -1377,7 +1377,7 @@ void processInputBuffer(client *c) { /* Determine request type when unknown. */ if (!c->reqtype) { - if (c->querybuf[0] == '*') { + if (c->querybuf[c->qb_pos] == '*') { c->reqtype = PROTO_REQ_MULTIBULK; } else { c->reqtype = PROTO_REQ_INLINE; @@ -1400,7 +1400,7 @@ void processInputBuffer(client *c) { if (processCommand(c) == C_OK) { if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { /* Update the applied replication offset of our master. */ - c->reploff = c->read_reploff - sdslen(c->querybuf); + c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; } /* Don't reset the client structure for clients blocked in a @@ -1416,6 +1416,13 @@ void processInputBuffer(client *c) { if (server.current_client == NULL) break; } } + + /* Trim to pos */ + if (c->qb_pos) { + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; + } + server.current_client = NULL; } diff --git a/src/server.h b/src/server.h index 186d08250..ce127b587 100644 --- a/src/server.h +++ b/src/server.h @@ -710,6 +710,7 @@ typedef struct client { redisDb *db; /* Pointer to currently SELECTed DB. */ robj *name; /* As set by CLIENT SETNAME. */ sds querybuf; /* Buffer we use to accumulate client queries. */ + size_t qb_pos; /* The position we have read in querybuf. */ sds pending_querybuf; /* If this client is flagged as master, this buffer represents the yet not applied portion of the replication stream that we are receiving from From b89302c462bfa410977ed99e1e6fdc64a0fbd031 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 00:57:22 +0800 Subject: [PATCH 084/616] adjust qbuf to 26 in test case for client list --- tests/unit/introspection.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index f6477d9c5..2581eb83d 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1,7 +1,7 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} test {MONITOR can log executed commands} { set rd [redis_deferring_client] From e623bd22bab45d8ce20d82bb2de5e8ed82f9e7bf Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 13:55:30 +0800 Subject: [PATCH 085/616] networking: just return C_OK if multibulk processing saw a <= 0 length. --- src/networking.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 945fdd961..9e1a3a9ef 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1252,11 +1252,8 @@ int processMultibulkBuffer(client *c) { } c->qb_pos = (newline-c->querybuf)+2; - if (ll <= 0) { - sdsrange(c->querybuf,c->qb_pos,-1); - c->qb_pos = 0; - return C_OK; - } + + if (ll <= 0) return C_OK; c->multibulklen = ll; From ef2a95c46125a5e5402a2ee1e433f78959596d0c Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 14:50:37 +0800 Subject: [PATCH 086/616] networking: just move qb_pos instead of sdsrange in processInlineBuffer --- src/networking.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 9e1a3a9ef..24432339b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1152,9 +1152,8 @@ int processInlineBuffer(client *c) { if (querylen == 0 && c->flags & CLIENT_SLAVE) c->repl_ack_time = server.unixtime; - /* Leave data after the first line of the query in the buffer */ - sdsrange(c->querybuf,c->qb_pos+querylen+linefeed_chars,-1); - c->qb_pos = 0; + /* Move querybuffer position to the next query in the buffer. */ + c->qb_pos += querylen+linefeed_chars; /* Setup argv array on client structure */ if (argc) { From 8a1219d93b3a9e3349c6f6726bf9923fcf2a40e5 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 20:58:58 +0800 Subject: [PATCH 087/616] block: rewrite BRPOPLPUSH as RPOPLPUSH to propagate --- src/server.c | 1 + src/server.h | 2 +- src/t_list.c | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index b537ee04a..cd88d3c2f 100644 --- a/src/server.c +++ b/src/server.c @@ -1491,6 +1491,7 @@ void createSharedObjects(void) { shared.rpop = createStringObject("RPOP",4); shared.lpop = createStringObject("LPOP",4); shared.lpush = createStringObject("LPUSH",5); + shared.rpoplpush = createStringObject("RPOPLPUSH",9); shared.zpopmin = createStringObject("ZPOPMIN",7); shared.zpopmax = createStringObject("ZPOPMAX",7); for (j = 0; j < OBJ_SHARED_INTEGERS; j++) { diff --git a/src/server.h b/src/server.h index 186d08250..05be23c9a 100644 --- a/src/server.h +++ b/src/server.h @@ -781,7 +781,7 @@ struct sharedObjectsStruct { *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, - *rpop, *lpop, *lpush, *zpopmin, *zpopmax, *emptyscan, + *rpop, *lpop, *lpush, *rpoplpush, *zpopmin, *zpopmax, *emptyscan, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*\r\n" */ diff --git a/src/t_list.c b/src/t_list.c index 1414ff31a..987392e6e 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -596,6 +596,9 @@ void rpoplpushCommand(client *c) { signalModifiedKey(c->db,touchedkey); decrRefCount(touchedkey); server.dirty++; + if (c->lastcmd->proc == brpoplpushCommand) { + rewriteClientCommandVector(c,3,shared.rpoplpush,c->argv[1],c->argv[2]); + } } } From 9a65f9cd3e09468c0178782bfac0fe1c8f445a20 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 20:59:32 +0800 Subject: [PATCH 088/616] block: format code --- src/blocked.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 4a667501f..35c33d1cc 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -269,7 +269,7 @@ void handleClientsBlockedOnKeys(void) { robj *dstkey = receiver->bpop.target; int where = (receiver->lastcmd && receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; + LIST_HEAD : LIST_TAIL; robj *value = listTypePop(o,where); if (value) { @@ -285,7 +285,7 @@ void handleClientsBlockedOnKeys(void) { { /* If we failed serving the client we need * to also undo the POP operation. */ - listTypePush(o,value,where); + listTypePush(o,value,where); } if (dstkey) decrRefCount(dstkey); From c8452ab0059877ac4a055e74e604d00244b06053 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 21 Aug 2018 11:46:07 +0300 Subject: [PATCH 089/616] Fix unstable tests on slow machines. Few tests had borderline thresholds that were adjusted. The slave buffers test had two issues, preventing the slave buffer from growing: 1) the slave didn't necessarily go to sleep on time, or woke up too early, now using SIGSTOP to make sure it goes to sleep exactly when we want. 2) the master disconnected the slave on timeout --- tests/unit/dump.tcl | 4 ++-- tests/unit/maxmemory.tcl | 34 ++++++++++++++++++++-------------- tests/unit/memefficiency.tcl | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index 91a4df09e..fa4f53b3b 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -33,7 +33,7 @@ start_server {tags {"dump"}} { set now [clock milliseconds] r restore foo [expr $now+3000] $encoded absttl set ttl [r pttl foo] - assert {$ttl >= 2998 && $ttl <= 3000} + assert {$ttl >= 2990 && $ttl <= 3000} r get foo } {bar} @@ -44,7 +44,7 @@ start_server {tags {"dump"}} { r config set maxmemory-policy allkeys-lru r restore foo 0 $encoded idletime 1000 set idle [r object idletime foo] - assert {$idle >= 1000 && $idle <= 1002} + assert {$idle >= 1000 && $idle <= 1010} r get foo } {bar} diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index 7629fe05e..f6d596ca2 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -143,9 +143,11 @@ start_server {tags {"maxmemory"}} { } } -proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { +proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} { start_server {tags {"maxmemory"}} { start_server {} { + set slave_pid [s process_id] + test "$test_name" { set slave [srv 0 client] set slave_host [srv 0 host] set slave_port [srv 0 port] @@ -158,8 +160,10 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { $master setrange "key:$j" 100000 asdf } + # make sure master doesn't disconnect slave because of timeout + $master config set repl-timeout 300 ;# 5 minutes $master config set maxmemory-policy allkeys-random - $master config set client-output-buffer-limit "slave 100000000 100000000 60" + $master config set client-output-buffer-limit "slave 100000000 100000000 300" $master config set repl-backlog-size [expr {10*1024}] $slave slaveof $master_host $master_port @@ -182,9 +186,9 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { # put the slave to sleep set rd_slave [redis_deferring_client] - $rd_slave debug sleep 300 + exec kill -SIGSTOP $slave_pid - # send some 10mb woth of commands that don't increase the memory usage + # send some 10mb worth of commands that don't increase the memory usage if {$pipeline == 1} { set rd_master [redis_deferring_client -1] for {set k 0} {$k < $cmd_count} {incr k} { @@ -218,19 +222,21 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { set delta_no_repl [expr {$killed_used_no_repl - $used_no_repl}] assert {$killed_slave_buf == 0} assert {$delta_no_repl > -50*1024 && $delta_no_repl < 50*1024} ;# 1 byte unaccounted for, with 1M commands will consume some 1MB + + } + # unfreeze slave process (after the 'test' succeeded or failed, but before we attempt to terminate the server + exec kill -SIGCONT $slave_pid } } } -test {slave buffer are counted correctly} { - # we wanna use many small commands, and we don't wanna wait long - # so we need to use a pipeline (redis_deferring_client) - # that may cause query buffer to fill and induce eviction, so we disable it - test_slave_buffers 1000000 10 0 1 -} +# test that slave buffer are counted correctly +# we wanna use many small commands, and we don't wanna wait long +# so we need to use a pipeline (redis_deferring_client) +# that may cause query buffer to fill and induce eviction, so we disable it +test_slave_buffers {slave buffer are counted correctly} 1000000 10 0 1 -test {slave buffer don't induce eviction} { - # test again with fewer (and bigger) commands without pipeline, but with eviction - test_slave_buffers 100000 100 1 0 -} +# test that slave buffer don't induce eviction +# test again with fewer (and bigger) commands without pipeline, but with eviction +test_slave_buffers "slave buffer don't induce eviction" 100000 100 1 0 diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index ec71a36b1..8972d577a 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -201,7 +201,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 10hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 75ms - assert {$max_latency <= 80} + assert {$max_latency <= 120} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From 1ab64d405e66c8fecd0ff7bd65d22c46f2b1ad0e Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Wed, 22 Aug 2018 18:07:02 +0800 Subject: [PATCH 090/616] Revise the comments of latency command. --- src/latency.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/latency.c b/src/latency.c index e8d2af306..d89c48db1 100644 --- a/src/latency.c +++ b/src/latency.c @@ -560,10 +560,11 @@ sds latencyCommandGenSparkeline(char *event, struct latencyTimeSeries *ts) { /* LATENCY command implementations. * - * LATENCY SAMPLES: return time-latency samples for the specified event. + * LATENCY HISTORY: return time-latency samples for the specified event. * LATENCY LATEST: return the latest latency for all the events classes. * LATENCY DOCTOR: returns an human readable analysis of instance latency. * LATENCY GRAPH: provide an ASCII graph of the latency of the specified event. + * LATENCY RESET: reset data of a specified event or all the data if no event provided. */ void latencyCommand(client *c) { struct latencyTimeSeries *ts; From f2ad89a314a0be2ea50a598339ee903ec3f64b65 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 23 Aug 2018 12:21:23 +0800 Subject: [PATCH 091/616] networking: make setProtocolError simple and clear Function setProtocolError just records proctocol error details in server log, set client as CLIENT_CLOSE_AFTER_REPLY. It doesn't care about querybuf sdsrange, because we will do it after procotol parsing. --- src/networking.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/networking.c b/src/networking.c index 24432339b..58248ced0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -33,7 +33,7 @@ #include #include -static void setProtocolError(const char *errstr, client *c, long pos); +static void setProtocolError(const char *errstr, client *c); /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -1126,7 +1126,7 @@ int processInlineBuffer(client *c) { if (newline == NULL) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big inline request"); - setProtocolError("too big inline request",c,0); + setProtocolError("too big inline request",c); } return C_ERR; } @@ -1142,7 +1142,7 @@ int processInlineBuffer(client *c) { sdsfree(aux); if (argv == NULL) { addReplyError(c,"Protocol error: unbalanced quotes in request"); - setProtocolError("unbalanced quotes in inline request",c,0); + setProtocolError("unbalanced quotes in inline request",c); return C_ERR; } @@ -1174,10 +1174,10 @@ int processInlineBuffer(client *c) { return C_OK; } -/* Helper function. Trims query buffer to make the function that processes - * multi bulk requests idempotent. */ +/* Helper function. Record protocol erro details in server log, + * and set the client as CLIENT_CLOSE_AFTER_REPLY. */ #define PROTO_DUMP_LEN 128 -static void setProtocolError(const char *errstr, client *c, long pos) { +static void setProtocolError(const char *errstr, client *c) { if (server.verbosity <= LL_VERBOSE) { sds client = catClientInfoString(sdsempty(),c); @@ -1202,8 +1202,6 @@ static void setProtocolError(const char *errstr, client *c, long pos) { sdsfree(client); } c->flags |= CLIENT_CLOSE_AFTER_REPLY; - sdsrange(c->querybuf,pos,-1); - c->qb_pos -= pos; } /* Process the query buffer for client 'c', setting up the client argument @@ -1231,7 +1229,7 @@ int processMultibulkBuffer(client *c) { if (newline == NULL) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big mbulk count string"); - setProtocolError("too big mbulk count string",c,0); + setProtocolError("too big mbulk count string",c); } return C_ERR; } @@ -1246,7 +1244,7 @@ int processMultibulkBuffer(client *c) { ok = string2ll(c->querybuf+1+c->qb_pos,newline-(c->querybuf+1+c->qb_pos),&ll); if (!ok || ll > 1024*1024) { addReplyError(c,"Protocol error: invalid multibulk length"); - setProtocolError("invalid mbulk count",c,c->qb_pos); + setProtocolError("invalid mbulk count",c); return C_ERR; } @@ -1270,7 +1268,7 @@ int processMultibulkBuffer(client *c) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c, "Protocol error: too big bulk count string"); - setProtocolError("too big bulk count string",c,0); + setProtocolError("too big bulk count string",c); return C_ERR; } break; @@ -1284,14 +1282,14 @@ int processMultibulkBuffer(client *c) { addReplyErrorFormat(c, "Protocol error: expected '$', got '%c'", c->querybuf[c->qb_pos]); - setProtocolError("expected $ but got something else",c,c->qb_pos); + setProtocolError("expected $ but got something else",c); return C_ERR; } ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll); if (!ok || ll < 0 || ll > server.proto_max_bulk_len) { addReplyError(c,"Protocol error: invalid bulk length"); - setProtocolError("invalid bulk length",c,c->qb_pos); + setProtocolError("invalid bulk length",c); return C_ERR; } From 132be8aed5c7669f3061d2861eb2570c9f41dc19 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sun, 26 Aug 2018 14:45:39 +0200 Subject: [PATCH 092/616] Correct "did not received" -> "did not receive" typos/grammar. --- src/scripting.c | 2 +- src/sentinel.c | 6 +++--- tests/sentinel/tests/00-base.tcl | 4 ++-- tests/sentinel/tests/01-conf-update.tcl | 4 ++-- tests/sentinel/tests/02-slaves-reconf.tcl | 2 +- tests/sentinel/tests/05-manual.tcl | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 2732c87fb..9afc08a94 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1725,7 +1725,7 @@ int ldbRemoveChild(pid_t pid) { return 0; } -/* Return the number of children we still did not received termination +/* Return the number of children we still did not receive termination * acknowledge via wait() in the parent process. */ int ldbPendingChildren(void) { return listLength(ldb.children); diff --git a/src/sentinel.c b/src/sentinel.c index 76e0eb750..b608fc693 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2628,7 +2628,7 @@ int sentinelSendPing(sentinelRedisInstance *ri) { ri->link->last_ping_time = mstime(); /* We update the active ping time only if we received the pong for * the previous ping, otherwise we are technically waiting since the - * first ping that did not received a reply. */ + * first ping that did not receive a reply. */ if (ri->link->act_ping_time == 0) ri->link->act_ping_time = ri->link->last_ping_time; return 1; @@ -3569,7 +3569,7 @@ void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) { (mstime() - ri->link->cc_conn_time) > SENTINEL_MIN_LINK_RECONNECT_PERIOD && ri->link->act_ping_time != 0 && /* There is a pending ping... */ - /* The pending ping is delayed, and we did not received + /* The pending ping is delayed, and we did not receive * error replies as well. */ (mstime() - ri->link->act_ping_time) > (ri->down_after_period/2) && (mstime() - ri->link->last_pong_time) > (ri->down_after_period/2)) @@ -3725,7 +3725,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f * * 1) We believe it is down, or there is a failover in progress. * 2) Sentinel is connected. - * 3) We did not received the info within SENTINEL_ASK_PERIOD ms. */ + * 3) We did not receive the info within SENTINEL_ASK_PERIOD ms. */ if ((master->flags & SRI_S_DOWN) == 0) continue; if (ri->link->disconnected) continue; if (!(flags & SENTINEL_ASK_FORCED) && diff --git a/tests/sentinel/tests/00-base.tcl b/tests/sentinel/tests/00-base.tcl index a79d0c371..7fb1a8bef 100644 --- a/tests/sentinel/tests/00-base.tcl +++ b/tests/sentinel/tests/00-base.tcl @@ -17,7 +17,7 @@ test "Basic failover works if the master is down" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id @@ -108,7 +108,7 @@ test "Failover works if we configure for absolute agreement" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id diff --git a/tests/sentinel/tests/01-conf-update.tcl b/tests/sentinel/tests/01-conf-update.tcl index 4998104d2..d45b1b08e 100644 --- a/tests/sentinel/tests/01-conf-update.tcl +++ b/tests/sentinel/tests/01-conf-update.tcl @@ -16,7 +16,7 @@ test "We can failover with Sentinel 1 crashed" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "Sentinel $id did not received failover info" + fail "Sentinel $id did not receive failover info" } } } @@ -30,7 +30,7 @@ test "After Sentinel 1 is restarted, its config gets updated" { wait_for_condition 1000 50 { [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "Restarted Sentinel did not received failover info" + fail "Restarted Sentinel did not receive failover info" } } diff --git a/tests/sentinel/tests/02-slaves-reconf.tcl b/tests/sentinel/tests/02-slaves-reconf.tcl index fa15d2efd..28964c968 100644 --- a/tests/sentinel/tests/02-slaves-reconf.tcl +++ b/tests/sentinel/tests/02-slaves-reconf.tcl @@ -36,7 +36,7 @@ proc 02_crash_and_failover {} { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id diff --git a/tests/sentinel/tests/05-manual.tcl b/tests/sentinel/tests/05-manual.tcl index 5214fdce1..ed568aa03 100644 --- a/tests/sentinel/tests/05-manual.tcl +++ b/tests/sentinel/tests/05-manual.tcl @@ -12,7 +12,7 @@ test "Manual failover works" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] From e711e6f614c71187ca39b5b171f063a2025de1a0 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 27 Aug 2018 12:07:24 +0800 Subject: [PATCH 093/616] remove duplicate bind in sentinel.conf --- sentinel.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sentinel.conf b/sentinel.conf index 551defef9..b7fe7bb63 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -30,16 +30,6 @@ daemonize no # location here. pidfile /var/run/redis-sentinel.pid -# By default Redis Sentinel listens for connections from all the network -# interfaces available on the server. It is possible to listen to just one or -# multiple interfaces using the "bind" configuration directive, followed by one -# or more IP addresses. -# -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 - # Specify the log file name. Also the empty string can be used to force # Sentinel to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null From c241f5160752753c271366547e2ada8617f1e684 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:14:10 +0200 Subject: [PATCH 094/616] Fix build errors caused by #2358. --- src/config.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/config.c b/src/config.c index 0bf2f5f41..7bd9592b2 100644 --- a/src/config.c +++ b/src/config.c @@ -887,17 +887,6 @@ void configSetCommand(client *c) { } config_set_special_field("masterauth") { zfree(server.masterauth); server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { - if (getLongLongFromObject(o,&ll) == REDIS_ERR || - ll < 0) goto badfmt; - server.maxmemory = ll; - if (server.maxmemory) { - if (server.maxmemory < zmalloc_used_memory()) { - redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeeded(); - } - } else if (!strcasecmp(c->argv[2]->ptr,"maxclients")) { } config_set_special_field("cluster-announce-ip") { zfree(server.cluster_announce_ip); server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; From 66b5afdaa4bf53b66e4d09fe6c3ebbfff283d1c4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:17:14 +0200 Subject: [PATCH 095/616] Re-apply rebased #2358. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 7bd9592b2..b15cb6d77 100644 --- a/src/config.c +++ b/src/config.c @@ -1181,7 +1181,7 @@ void configSetCommand(client *c) { } config_set_memory_field("maxmemory",server.maxmemory) { if (server.maxmemory) { if (server.maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); } freeMemoryIfNeeded(); } From abf52c7cf412e2b7fd1235cb85248d23db295b48 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 11:57:44 +0200 Subject: [PATCH 096/616] Better variable meaning in processCommand(). --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index cd88d3c2f..aeb30cdf1 100644 --- a/src/server.c +++ b/src/server.c @@ -2594,14 +2594,14 @@ int processCommand(client *c) { * keys in the dataset). If there are not the only thing we can do * is returning an error. */ if (server.maxmemory) { - int retval = freeMemoryIfNeeded(); + int out_of_memory = freeMemoryIfNeeded() == C_ERR; /* freeMemoryIfNeeded may flush slave output buffers. This may result * into a slave, that may be the active client, to be freed. */ if (server.current_client == NULL) return C_ERR; /* It was impossible to free enough memory, and the command the client * is trying to execute is denied during OOM conditions? Error. */ - if ((c->cmd->flags & CMD_DENYOOM) && retval == C_ERR) { + if ((c->cmd->flags & CMD_DENYOOM) && out_of_memory) { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; From 067647a78315cfa81bf489128afd6f4ae1388603 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:09:08 +0200 Subject: [PATCH 097/616] Introduce repl_slave_ignore_maxmemory flag internally. Note: this breaks backward compatibility with Redis 4, since now slaves by default are exact copies of masters and do not try to evict keys independently. --- src/evict.c | 4 ++++ src/server.c | 1 + src/server.h | 2 ++ 3 files changed, 7 insertions(+) diff --git a/src/evict.c b/src/evict.c index ecc25dd8e..cdb49a551 100644 --- a/src/evict.c +++ b/src/evict.c @@ -444,6 +444,10 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * Otehrwise if we are over the memory limit, but not enough memory * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { + /* By default slaves should ignore maxmemory and just be masters excat + * copies. */ + if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; + size_t mem_reported, mem_tofree, mem_freed; mstime_t latency, eviction_latency; long long delta; diff --git a/src/server.c b/src/server.c index aeb30cdf1..4aa36c287 100644 --- a/src/server.c +++ b/src/server.c @@ -1649,6 +1649,7 @@ void initServerConfig(void) { server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA; server.repl_slave_ro = CONFIG_DEFAULT_SLAVE_READ_ONLY; + server.repl_slave_ignore_maxmemory = CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY; server.repl_slave_lazy_flush = CONFIG_DEFAULT_SLAVE_LAZY_FLUSH; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.repl_disable_tcp_nodelay = CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY; diff --git a/src/server.h b/src/server.h index 2e1f7c9ce..4fb8e383c 100644 --- a/src/server.h +++ b/src/server.h @@ -131,6 +131,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY 5 #define CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA 1 #define CONFIG_DEFAULT_SLAVE_READ_ONLY 1 +#define CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY 1 #define CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP NULL #define CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT 0 #define CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 @@ -1157,6 +1158,7 @@ struct redisServer { time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */ int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ + int repl_slave_ignore_maxmemory; /* If true slaves do not evict. */ time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ From e245a2046af680a18f74cb76c2a23f5711149501 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:27:17 +0200 Subject: [PATCH 098/616] Make slave-ignore-maxmemory configurable. --- src/config.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config.c b/src/config.c index b15cb6d77..1a64d25ed 100644 --- a/src/config.c +++ b/src/config.c @@ -399,6 +399,10 @@ void loadServerConfigFromString(char *config) { if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-ignore-maxmemory") && argc == 2) { + if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -1047,6 +1051,8 @@ void configSetCommand(client *c) { "slave-serve-stale-data",server.repl_serve_stale_data) { } config_set_bool_field( "slave-read-only",server.repl_slave_ro) { + } config_set_bool_field( + "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { } config_set_bool_field( "activerehashing",server.activerehashing) { } config_set_bool_field( @@ -1353,6 +1359,8 @@ void configGetCommand(client *c) { server.repl_serve_stale_data); config_get_bool_field("slave-read-only", server.repl_slave_ro); + config_get_bool_field("slave-ignore-maxmemory", + server.repl_slave_ignore_maxmemory); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); @@ -2049,6 +2057,7 @@ int rewriteConfig(char *path) { rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); + rewriteConfigYesNoOption(state,"slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); From 5941022b524bd8b7017af3811083ed42a85de000 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:34:29 +0200 Subject: [PATCH 099/616] Document slave-ignore-maxmemory in redis.conf. --- redis.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/redis.conf b/redis.conf index 5cbc74bbd..886c64f21 100644 --- a/redis.conf +++ b/redis.conf @@ -602,6 +602,26 @@ slave-priority 100 # # maxmemory-samples 5 +# Starting from Redis 5, by default a slave will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the slave as keys evict in the master side. +# +# This behavior ensures that masters and slaves stay consistent, and is usually +# what you want, however if your slave is writable, or you want the slave to have +# a different memory setting, and you are sure all the writes performed to the +# slave are idempotent, then you may change this default (but be sure to understand +# what you are doing). +# +# Note that since the slave by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the slave, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your slaves and make sure they have enough +# memory to never hit a real out-of-memory condition before the master hits +# the configured maxmemory setting. +# +# slave-ingore-maxmemory yes + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking From 32844178ac711535903eb4ecbd6d2774200ea951 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 10 Apr 2018 20:41:21 +0800 Subject: [PATCH 100/616] some commands' flags should be set correctly, issue #4834 --- src/server.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/server.c b/src/server.c index 7f51778d7..79227142e 100644 --- a/src/server.c +++ b/src/server.c @@ -210,7 +210,7 @@ struct redisCommand redisCommandTable[] = { {"hstrlen",hstrlenCommand,3,"rF",0,NULL,1,1,1,0,0}, {"hkeys",hkeysCommand,2,"rS",0,NULL,1,1,1,0,0}, {"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0}, - {"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0}, + {"hgetall",hgetallCommand,2,"rR",0,NULL,1,1,1,0,0}, {"hexists",hexistsCommand,3,"rF",0,NULL,1,1,1,0,0}, {"hscan",hscanCommand,-3,"rR",0,NULL,1,1,1,0,0}, {"incrby",incrbyCommand,3,"wmF",0,NULL,1,1,1,0,0}, @@ -236,9 +236,9 @@ struct redisCommand redisCommandTable[] = { {"ping",pingCommand,-1,"tF",0,NULL,0,0,0,0,0}, {"echo",echoCommand,2,"F",0,NULL,0,0,0,0,0}, {"save",saveCommand,1,"as",0,NULL,0,0,0,0,0}, - {"bgsave",bgsaveCommand,-1,"a",0,NULL,0,0,0,0,0}, - {"bgrewriteaof",bgrewriteaofCommand,1,"a",0,NULL,0,0,0,0,0}, - {"shutdown",shutdownCommand,-1,"alt",0,NULL,0,0,0,0,0}, + {"bgsave",bgsaveCommand,-1,"as",0,NULL,0,0,0,0,0}, + {"bgrewriteaof",bgrewriteaofCommand,1,"as",0,NULL,0,0,0,0,0}, + {"shutdown",shutdownCommand,-1,"aslt",0,NULL,0,0,0,0,0}, {"lastsave",lastsaveCommand,1,"RF",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"rF",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"sF",0,NULL,0,0,0,0,0}, @@ -250,16 +250,16 @@ struct redisCommand redisCommandTable[] = { {"flushdb",flushdbCommand,-1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,-1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wm",0,sortGetKeys,1,1,1,0,0}, - {"info",infoCommand,-1,"lt",0,NULL,0,0,0,0,0}, + {"info",infoCommand,-1,"lts",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"as",0,NULL,0,0,0,0,0}, - {"ttl",ttlCommand,2,"rF",0,NULL,1,1,1,0,0}, + {"ttl",ttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, - {"pttl",pttlCommand,2,"rF",0,NULL,1,1,1,0,0}, + {"pttl",pttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"wF",0,NULL,1,1,1,0,0}, {"slaveof",slaveofCommand,3,"ast",0,NULL,0,0,0,0,0}, {"role",roleCommand,1,"lst",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, - {"config",configCommand,-2,"lat",0,NULL,0,0,0,0,0}, + {"config",configCommand,-2,"last",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"pslt",0,NULL,0,0,0,0,0}, {"unsubscribe",unsubscribeCommand,-1,"pslt",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"pslt",0,NULL,0,0,0,0,0}, @@ -271,24 +271,24 @@ struct redisCommand redisCommandTable[] = { {"cluster",clusterCommand,-2,"a",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,-4,"wm",0,NULL,1,1,1,0,0}, {"restore-asking",restoreCommand,-4,"wmk",0,NULL,1,1,1,0,0}, - {"migrate",migrateCommand,-6,"w",0,migrateGetKeys,0,0,0,0,0}, + {"migrate",migrateCommand,-6,"wR",0,migrateGetKeys,0,0,0,0,0}, {"asking",askingCommand,1,"F",0,NULL,0,0,0,0,0}, {"readonly",readonlyCommand,1,"F",0,NULL,0,0,0,0,0}, {"readwrite",readwriteCommand,1,"F",0,NULL,0,0,0,0,0}, - {"dump",dumpCommand,2,"r",0,NULL,1,1,1,0,0}, - {"object",objectCommand,-2,"r",0,NULL,2,2,1,0,0}, - {"memory",memoryCommand,-2,"r",0,NULL,0,0,0,0,0}, + {"dump",dumpCommand,2,"rR",0,NULL,1,1,1,0,0}, + {"object",objectCommand,-2,"rR",0,NULL,2,2,1,0,0}, + {"memory",memoryCommand,-2,"rs",0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,"as",0,NULL,0,0,0,0,0}, {"eval",evalCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, {"evalsha",evalShaCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, - {"slowlog",slowlogCommand,-2,"a",0,NULL,0,0,0,0,0}, + {"slowlog",slowlogCommand,-2,"aR",0,NULL,0,0,0,0,0}, {"script",scriptCommand,-2,"s",0,NULL,0,0,0,0,0}, {"time",timeCommand,1,"RF",0,NULL,0,0,0,0,0}, {"bitop",bitopCommand,-4,"wm",0,NULL,2,-1,1,0,0}, {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0}, {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0}, {"wait",waitCommand,3,"s",0,NULL,0,0,0,0,0}, - {"command",commandCommand,0,"lt",0,NULL,0,0,0,0,0}, + {"command",commandCommand,0,"lts",0,NULL,0,0,0,0,0}, {"geoadd",geoaddCommand,-5,"wm",0,NULL,1,1,1,0,0}, {"georadius",georadiusCommand,-6,"w",0,georadiusGetKeys,1,1,1,0,0}, {"georadius_ro",georadiusroCommand,-6,"r",0,georadiusGetKeys,1,1,1,0,0}, From 7d39c149c4ad3cf09fe58f2639ca58b4353294f0 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 29 Aug 2018 18:23:05 +0800 Subject: [PATCH 101/616] Supplement to PR #4835, just take info/memory/command as random commands --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index c8f9ef021..55bcb4dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -254,7 +254,7 @@ struct redisCommand redisCommandTable[] = { {"flushdb",flushdbCommand,-1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,-1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wm",0,sortGetKeys,1,1,1,0,0}, - {"info",infoCommand,-1,"lts",0,NULL,0,0,0,0,0}, + {"info",infoCommand,-1,"ltR",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"as",0,NULL,0,0,0,0,0}, {"ttl",ttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, @@ -281,7 +281,7 @@ struct redisCommand redisCommandTable[] = { {"readwrite",readwriteCommand,1,"F",0,NULL,0,0,0,0,0}, {"dump",dumpCommand,2,"rR",0,NULL,1,1,1,0,0}, {"object",objectCommand,-2,"rR",0,NULL,2,2,1,0,0}, - {"memory",memoryCommand,-2,"rs",0,NULL,0,0,0,0,0}, + {"memory",memoryCommand,-2,"rR",0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,"as",0,NULL,0,0,0,0,0}, {"eval",evalCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, {"evalsha",evalShaCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, @@ -292,7 +292,7 @@ struct redisCommand redisCommandTable[] = { {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0}, {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0}, {"wait",waitCommand,3,"s",0,NULL,0,0,0,0,0}, - {"command",commandCommand,0,"lts",0,NULL,0,0,0,0,0}, + {"command",commandCommand,0,"ltR",0,NULL,0,0,0,0,0}, {"geoadd",geoaddCommand,-5,"wm",0,NULL,1,1,1,0,0}, {"georadius",georadiusCommand,-6,"w",0,georadiusGetKeys,1,1,1,0,0}, {"georadius_ro",georadiusroCommand,-6,"r",0,georadiusGetKeys,1,1,1,0,0}, From dce7cefb7ca5b8efad865fca14c16d773a106e3d Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 31 Aug 2018 11:49:27 +0800 Subject: [PATCH 102/616] networking: fix unexpected negative or zero readlen To avoid copying buffers to create a large Redis Object which exceeding PROTO_IOBUF_LEN 32KB, we just read the remaining data we need, which may less than PROTO_IOBUF_LEN. But the remaining len may be zero, if the bulklen+2 equals sdslen(c->querybuf), in client pause context. For example: Time1: python >>> import os, socket >>> server="127.0.0.1" >>> port=6379 >>> data1="*3\r\n$3\r\nset\r\n$1\r\na\r\n$33000\r\n" >>> data2="".join("x" for _ in range(33000)) + "\r\n" >>> data3="\n\n" >>> s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) >>> s.settimeout(10) >>> s.connect((server, port)) >>> s.send(data1) 28 Time2: redis-cli client pause 10000 Time3: >>> s.send(data2) 33002 >>> s.send(data3) 2 >>> s.send(data3) Traceback (most recent call last): File "", line 1, in socket.error: [Errno 104] Connection reset by peer To fix that, we should check if remaining is greater than zero. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 58248ced0..17d82d1a2 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1439,7 +1439,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { { ssize_t remaining = (size_t)(c->bulklen+2)-sdslen(c->querybuf); - if (remaining < readlen) readlen = remaining; + if (remaining > 0 && remaining < readlen) readlen = remaining; } qblen = sdslen(c->querybuf); From f5b29c6444c85a56dc1c60a9c6ee308b779be838 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Aug 2018 18:14:46 +0200 Subject: [PATCH 103/616] Allow scripts to timeout on slaves as well. See reasoning in #5297. --- src/scripting.c | 12 +++++++++--- src/server.c | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 9afc08a94..c3ee0f971 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1222,12 +1222,18 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { /* This is the Lua script "count" hook that we use to detect scripts timeout. */ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed; + long long elapsed = mstime() - server.lua_time_start; UNUSED(ar); UNUSED(lua); - elapsed = mstime() - server.lua_time_start; - if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { + /* The conditions to timeout are: + * 1. The caller is not our master. + * 2. The timeout was reached. + * 3. We are not already timed out. */ + if (!(server.lua_caller->flags & CLIENT_MASTER) && + elapsed >= server.lua_time_limit && + server.lua_timedout == 0) + { serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others diff --git a/src/server.c b/src/server.c index 55bcb4dc2..1afc9a802 100644 --- a/src/server.c +++ b/src/server.c @@ -2683,6 +2683,7 @@ int processCommand(client *c) { /* Lua script too slow? Only allow a limited number of commands. */ if (server.lua_timedout && + !(c->flags & CLIENT_MASTER) && c->cmd->proc != authCommand && c->cmd->proc != replconfCommand && !(c->cmd->proc == shutdownCommand && From 83af8ef1fdbfb69dd53cedc8e772184b096a5da8 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Aug 2018 11:46:42 +0200 Subject: [PATCH 104/616] Allow scripts to timeout even if from the master instance. However the master scripts will be impossible to kill. Related to #5297. --- src/scripting.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index c3ee0f971..0ef8d2a6f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1226,14 +1226,9 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { UNUSED(ar); UNUSED(lua); - /* The conditions to timeout are: - * 1. The caller is not our master. - * 2. The timeout was reached. - * 3. We are not already timed out. */ - if (!(server.lua_caller->flags & CLIENT_MASTER) && - elapsed >= server.lua_time_limit && - server.lua_timedout == 0) - { + /* Set the timeout condition if not already set and the maximum + * execution time was reached. */ + if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others @@ -1351,9 +1346,7 @@ void evalGenericCommand(client *c, int evalsha) { server.lua_caller = c; server.lua_time_start = mstime(); server.lua_kill = 0; - if (server.lua_time_limit > 0 && server.masterhost == NULL && - ldb.active == 0) - { + if (server.lua_time_limit > 0 && ldb.active == 0) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); delhook = 1; } else if (ldb.active) { @@ -1503,6 +1496,8 @@ NULL } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { if (server.lua_caller == NULL) { addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n")); + } else if (server.lua_caller->flags & CLIENT_MASTER) { + addReplySds(c,sdsnew("-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed.\r\n")); } else if (server.lua_write_dirty) { addReplySds(c,sdsnew("-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command.\r\n")); } else { From 9ab91b8c6c599aedad1c5f0521e2036dcf1d0566 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Aug 2018 17:40:58 +0200 Subject: [PATCH 105/616] While the slave is busy, just accumulate master input. Processing command from the master while the slave is in busy state is not correct, however we cannot, also, just reply -BUSY to the replication stream commands from the master. The correct solution is to stop processing data from the master, but just accumulate the stream into the buffers and resume the processing later. Related to #5297. --- src/networking.c | 6 ++++++ src/server.c | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 58248ced0..eb581462b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1362,6 +1362,12 @@ void processInputBuffer(client *c) { /* Immediately abort if the client is in the middle of something. */ if (c->flags & CLIENT_BLOCKED) break; + /* Don't process input from the master while there is a busy script + * condition on the slave. We want just to accumulate the replication + * stream (instead of replying -BUSY like we do with other clients) and + * later resume the processing. */ + if (server.lua_timedout && c->flags & CLIENT_MASTER) break; + /* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is * written to the client. Make sure to not let the reply grow after * this flag has been set (i.e. don't process more commands). diff --git a/src/server.c b/src/server.c index 1afc9a802..55bcb4dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -2683,7 +2683,6 @@ int processCommand(client *c) { /* Lua script too slow? Only allow a limited number of commands. */ if (server.lua_timedout && - !(c->flags & CLIENT_MASTER) && c->cmd->proc != authCommand && c->cmd->proc != replconfCommand && !(c->cmd->proc == shutdownCommand && From 7fa493912e5de72edd7b1720d1a7d1f4287998f8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 Aug 2018 16:07:03 +0200 Subject: [PATCH 106/616] After slave Lua script leaves busy state, re-process the master buffer. Technically speaking we don't really need to put the master client in the clients that need to be processed, since in practice the PING commands from the master will take care, however it is conceptually more sane to do so. --- src/networking.c | 3 +-- src/scripting.c | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index eb581462b..27c695306 100644 --- a/src/networking.c +++ b/src/networking.c @@ -827,8 +827,7 @@ void freeClient(client *c) { serverLog(LL_WARNING,"Connection with master lost."); if (!(c->flags & (CLIENT_CLOSE_AFTER_REPLY| CLIENT_CLOSE_ASAP| - CLIENT_BLOCKED| - CLIENT_UNBLOCKED))) + CLIENT_BLOCKED))) { replicationCacheMaster(c); return; diff --git a/src/scripting.c b/src/scripting.c index 0ef8d2a6f..4b36a085b 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,6 +1367,10 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); + if (server.masterhost && server.master) { + server.master->flags |= CLIENT_UNBLOCKED; + listAddNodeTail(server.unblocked_clients,server.master); + } } server.lua_caller = NULL; From febe102bf6d94428779f3943aea5947893301912 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 Aug 2018 16:43:38 +0200 Subject: [PATCH 107/616] Test: processing of master stream in slave -BUSY state. See #5297. --- tests/integration/replication.tcl | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index e811cf0ee..56d2de1cc 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -266,3 +266,47 @@ foreach dl {no yes} { } } } + +start_server {tags {"repl"}} { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set load_handle0 [start_write_load $master_host $master_port 3] + start_server {} { + test "Master stream is correctly processed while the slave has a script in -BUSY state" { + set slave [srv 0 client] + puts [srv 0 port] + $slave config set lua-time-limit 500 + $slave slaveof $master_host $master_port + + # Wait for the slave to be online + wait_for_condition 500 100 { + [lindex [$slave role] 3] eq {connected} + } else { + fail "Slave still not connected after some time" + } + + # Wait some time to make sure the master is sending data + # to the slave. + after 5000 + + # Stop the ability of the slave to process data by sendig + # a script that will put it in BUSY state. + $slave eval {for i=1,3000000000 do end} 0 + + # Wait some time again so that more master stream will + # be processed. + after 2000 + + # Stop the write load + stop_write_load $load_handle0 + + # number of keys + wait_for_condition 500 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Different datasets between slave and master" + } + } + } +} From a036c64c01b38aceebfcfd467da04ed3f95fd363 Mon Sep 17 00:00:00 2001 From: Amin Mesbah Date: Sun, 2 Sep 2018 00:06:20 -0700 Subject: [PATCH 108/616] Use geohash limit defines in constraint check Slight robustness improvement, especially if the limit values are changed, as was suggested in antires/redis#4291 [1]. [1] https://github.com/antirez/redis/pull/4291 --- src/geohash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geohash.c b/src/geohash.c index b40282e76..db5ae025a 100644 --- a/src/geohash.c +++ b/src/geohash.c @@ -127,8 +127,8 @@ int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range, /* Return an error when trying to index outside the supported * constraints. */ - if (longitude > 180 || longitude < -180 || - latitude > 85.05112878 || latitude < -85.05112878) return 0; + if (longitude > GEO_LONG_MAX || longitude < GEO_LONG_MIN || + latitude > GEO_LAT_MAX || latitude < GEO_LAT_MIN) return 0; hash->bits = 0; hash->step = step; From e3dfd8c811e12b6b1d3f7febf732b23fdaab497b Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 6 Nov 2017 16:21:35 +0800 Subject: [PATCH 109/616] fix multiple unblock for clientsArePaused() --- src/networking.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 27c695306..36228d6f0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2126,9 +2126,9 @@ int clientsArePaused(void) { while ((ln = listNext(&li)) != NULL) { c = listNodeValue(ln); - /* Don't touch slaves and blocked clients. The latter pending - * requests be processed when unblocked. */ - if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue; + /* Don't touch slaves and blocked or unblocked clients. + * The latter pending requests be processed when unblocked. */ + if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED|CLIENT_UNBLOCKED)) continue; c->flags |= CLIENT_UNBLOCKED; listAddNodeTail(server.unblocked_clients,c); } From 2290c4bee1904d4418fd1fa9192a3fbc13bf38b0 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 3 Sep 2018 14:36:48 +0800 Subject: [PATCH 110/616] if master is already unblocked, do not unblock it twice --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 4b36a085b..6c311dbe0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,7 +1367,7 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); - if (server.masterhost && server.master) { + if (server.masterhost && server.master && !(server.master->flags & CLIENT_UNBLOCKED)) { server.master->flags |= CLIENT_UNBLOCKED; listAddNodeTail(server.unblocked_clients,server.master); } From 247d2a734b0434e0f461902f64d32cb6e587709c Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 4 Sep 2018 00:02:18 +0800 Subject: [PATCH 111/616] networking: optimize parsing large bulk greater than 32k If we are going to read a large object from network try to make it likely that it will start at c->querybuf boundary so that we can optimize object creation avoiding a large copy of data. But only when the data we have not parsed is less than or equal to ll+2. If the data length is greater than ll+2, trimming querybuf is just a waste of time, because at this time the querybuf contains not only our bulk. It's easy to reproduce the that: Time1: call `client pause 10000` on slave. Time2: redis-benchmark -t set -r 10000 -d 33000 -n 10000. Then slave hung after 10 seconds. --- src/networking.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/networking.c b/src/networking.c index 27c695306..eac94798c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1294,19 +1294,22 @@ int processMultibulkBuffer(client *c) { c->qb_pos = newline-c->querybuf+2; if (ll >= PROTO_MBULK_BIG_ARG) { - size_t qblen; - /* If we are going to read a large object from network * try to make it likely that it will start at c->querybuf * boundary so that we can optimize object creation - * avoiding a large copy of data. */ - sdsrange(c->querybuf,c->qb_pos,-1); - c->qb_pos = 0; - qblen = sdslen(c->querybuf); - /* Hint the sds library about the amount of bytes this string is - * going to contain. */ - if (qblen < (size_t)ll+2) - c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-qblen); + * avoiding a large copy of data. + * + * But only when the data we have not parsed is less than + * or equal to ll+2. If the data length is greater than + * ll+2, trimming querybuf is just a waste of time, because + * at this time the querybuf contains not only our bulk. */ + if (sdslen(c->querybuf)-c->qb_pos <= (size_t)ll+2) { + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; + /* Hint the sds library about the amount of bytes this string is + * going to contain. */ + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2); + } } c->bulklen = ll; } From 3e7349fdaf8cdbf96e595750034af43e6d6c56f0 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Sep 2018 18:17:25 +0200 Subject: [PATCH 112/616] Make pending buffer processing safe for CLIENT_MASTER client. Related to #5305. --- src/blocked.c | 2 +- src/networking.c | 32 ++++++++++++++++++++------------ src/server.h | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 35c33d1cc..aeca87a6e 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -126,7 +126,7 @@ void processUnblockedClients(void) { * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { if (c->querybuf && sdslen(c->querybuf) > 0) { - processInputBuffer(c); + processInputBufferAndReplicate(c); } } } diff --git a/src/networking.c b/src/networking.c index 27c695306..8e55ec902 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1425,6 +1425,25 @@ void processInputBuffer(client *c) { server.current_client = NULL; } +/* This is a wrapper for processInputBuffer that also cares about handling + * the replication forwarding to the sub-slaves, in case the client 'c' + * is flagged as master. Usually you want to call this instead of the + * raw processInputBuffer(). */ +void processInputBufferAndReplicate(client *c) { + if (!(c->flags & CLIENT_MASTER)) { + processInputBuffer(c); + } else { + size_t prev_offset = c->reploff; + processInputBuffer(c); + size_t applied = c->reploff - prev_offset; + if (applied) { + replicationFeedSlavesFromMasterStream(server.slaves, + c->pending_querybuf, applied); + sdsrange(c->pending_querybuf,applied,-1); + } + } +} + void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { client *c = (client*) privdata; int nread, readlen; @@ -1492,18 +1511,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { * was actually applied to the master state: this quantity, and its * corresponding part of the replication stream, will be propagated to * the sub-slaves and to the replication backlog. */ - if (!(c->flags & CLIENT_MASTER)) { - processInputBuffer(c); - } else { - size_t prev_offset = c->reploff; - processInputBuffer(c); - size_t applied = c->reploff - prev_offset; - if (applied) { - replicationFeedSlavesFromMasterStream(server.slaves, - c->pending_querybuf, applied); - sdsrange(c->pending_querybuf,applied,-1); - } - } + processInputBufferAndReplicate(c); } void getClientsMaxBuffers(unsigned long *longest_output_list, diff --git a/src/server.h b/src/server.h index 4fb8e383c..4a5967f10 100644 --- a/src/server.h +++ b/src/server.h @@ -1418,6 +1418,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask); void *addDeferredMultiBulkLength(client *c); void setDeferredMultiBulkLength(client *c, void *node, long length); void processInputBuffer(client *c); +void processInputBufferAndReplicate(client *c); void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask); From 6c001bfc0ddf8e7a93e86dad250e416166253d85 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Sep 2018 18:39:18 +0200 Subject: [PATCH 113/616] Unblocked clients API refactoring. See #4418. --- src/blocked.c | 32 ++++++++++++++++++++++++++------ src/networking.c | 9 ++++----- src/scripting.c | 6 ++---- src/server.h | 1 + 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index aeca87a6e..00212ed69 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -132,6 +132,31 @@ void processUnblockedClients(void) { } } +/* This function will schedule the client for reprocessing at a safe time. + * + * This is useful when a client was blocked for some reason (blocking opeation, + * CLIENT PAUSE, or whatever), because it may end with some accumulated query + * buffer that needs to be processed ASAP: + * + * 1. When a client is blocked, its readable handler is still active. + * 2. However in this case it only gets data into the query buffer, but the + * query is not parsed or executed once there is enough to proceed as + * usually (because the client is blocked... so we can't execute commands). + * 3. When the client is unblocked, without this function, the client would + * have to write some query in order for the readable handler to finally + * call processQueryBuffer*() on it. + * 4. With this function instead we can put the client in a queue that will + * process it for queries ready to be executed at a safe time. + */ +void queueClientForReprocessing(client *c) { + /* The client may already be into the unblocked list because of a previous + * blocking operation, don't add back it into the list multiple times. */ + if (!(c->flags & CLIENT_UNBLOCKED)) { + c->flags |= CLIENT_UNBLOCKED; + listAddNodeTail(server.unblocked_clients,c); + } +} + /* Unblock a client calling the right function depending on the kind * of operation the client is blocking for. */ void unblockClient(client *c) { @@ -152,12 +177,7 @@ void unblockClient(client *c) { server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; - /* The client may already be into the unblocked list because of a previous - * blocking operation, don't add back it into the list multiple times. */ - if (!(c->flags & CLIENT_UNBLOCKED)) { - c->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,c); - } + queueClientForReprocessing(c); } /* This function gets called when a blocked client timed out in order to diff --git a/src/networking.c b/src/networking.c index 824d88245..0c1b3016f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2134,11 +2134,10 @@ int clientsArePaused(void) { while ((ln = listNext(&li)) != NULL) { c = listNodeValue(ln); - /* Don't touch slaves and blocked or unblocked clients. - * The latter pending requests be processed when unblocked. */ - if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED|CLIENT_UNBLOCKED)) continue; - c->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,c); + /* Don't touch slaves and blocked clients. + * The latter pending requests will be processed when unblocked. */ + if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue; + queueClientForReprocessing(c); } } return server.clients_paused; diff --git a/src/scripting.c b/src/scripting.c index 6c311dbe0..8aa62071c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,10 +1367,8 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); - if (server.masterhost && server.master && !(server.master->flags & CLIENT_UNBLOCKED)) { - server.master->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,server.master); - } + if (server.masterhost && server.master) + queueClientForReprocessing(server.master); } server.lua_caller = NULL; diff --git a/src/server.h b/src/server.h index 4a5967f10..09348585b 100644 --- a/src/server.h +++ b/src/server.h @@ -1884,6 +1884,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body); void processUnblockedClients(void); void blockClient(client *c, int btype); void unblockClient(client *c); +void queueClientForReprocessing(client *c); void replyToBlockedClientTimedOut(client *c); int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit); void disconnectAllBlockedClients(void); From 227965221ab443f21b27fa0a3636d6232c496b45 Mon Sep 17 00:00:00 2001 From: maya-rv <42264341+maya-rv@users.noreply.github.com> Date: Tue, 4 Sep 2018 13:32:02 +0300 Subject: [PATCH 114/616] Fix typo --- tests/support/server.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 5578f1fd6..0edb25d8a 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -4,7 +4,7 @@ set ::valgrind_errors {} proc start_server_error {config_file error} { set err {} - append err "Cant' start the Redis server\n" + append err "Can't start the Redis server\n" append err "CONFIGURATION:" append err [exec cat $config_file] append err "\nERROR:" From c1e9186f069e0b3aa1507e0e86a16d5001ea5420 Mon Sep 17 00:00:00 2001 From: Sascha Roland Date: Wed, 29 Aug 2018 17:53:47 +0200 Subject: [PATCH 115/616] #5299 Fix blocking XREAD for streams that ran dry The conclusion, that a xread request can be answered syncronously in case that the stream's last_id is larger than the passed last-received-id parameter, assumes, that there must be entries present, which could be returned immediately. This assumption fails for empty streams that actually contained some entries which got removed by xdel, ... . As result, the client is answered synchronously with an empty result, instead of blocking for new entries to arrive. An additional check for a non-empty stream is required. --- src/t_stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 77fbf4645..40e0c84b8 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1436,7 +1436,7 @@ void xreadCommand(client *c) { * synchronously in case the group top item delivered is smaller * than what the stream has inside. */ streamID *last = &groups[i]->last_id; - if (streamCompareID(&s->last_id, last) > 0) { + if (s->length && (streamCompareID(&s->last_id, last) > 0)) { serve_synchronously = 1; *gt = *last; } @@ -1444,7 +1444,7 @@ void xreadCommand(client *c) { } else { /* For consumers without a group, we serve synchronously if we can * actually provide at least one item from the stream. */ - if (streamCompareID(&s->last_id, gt) > 0) { + if (s->length && (streamCompareID(&s->last_id, gt) > 0)) { serve_synchronously = 1; } } From 4e5e0d3719ffed0d3a687fa2082655f97e16167c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Sep 2018 13:29:24 +0200 Subject: [PATCH 116/616] Clarify why remaining may be zero in readQueryFromClient(). See #5304. --- src/networking.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/networking.c b/src/networking.c index 605621c1c..01307c7a3 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1466,6 +1466,8 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { { ssize_t remaining = (size_t)(c->bulklen+2)-sdslen(c->querybuf); + /* Note that the 'remaining' variable may be zero in some edge case, + * for example once we resume a blocked client after CLIENT PAUSE. */ if (remaining > 0 && remaining < readlen) readlen = remaining; } From a8322f44b3f537a58d6e8a63dd02c66e2d7ed2e4 Mon Sep 17 00:00:00 2001 From: youjiali1995 Date: Wed, 5 Sep 2018 10:32:18 +0800 Subject: [PATCH 117/616] sentinel: fix randomized sentinelTimer. --- src/server.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 55bcb4dc2..cff218446 100644 --- a/src/server.c +++ b/src/server.c @@ -1320,9 +1320,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } /* Run the Sentinel timer if we are in sentinel mode. */ - run_with_period(100) { - if (server.sentinel_mode) sentinelTimer(); - } + if (server.sentinel_mode) sentinelTimer(); /* Cleanup expired MIGRATE cached sockets. */ run_with_period(1000) { From c328834832808cf8be9c30122a48b8234ebebed1 Mon Sep 17 00:00:00 2001 From: youjiali1995 Date: Wed, 5 Sep 2018 16:51:13 +0800 Subject: [PATCH 118/616] bio: fix bioWaitStepOfType. --- src/bio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bio.c b/src/bio.c index 0c92d053b..0e15c416d 100644 --- a/src/bio.c +++ b/src/bio.c @@ -204,14 +204,14 @@ void *bioProcessBackgroundJobs(void *arg) { } zfree(job); - /* Unblock threads blocked on bioWaitStepOfType() if any. */ - pthread_cond_broadcast(&bio_step_cond[type]); - /* Lock again before reiterating the loop, if there are no longer * jobs to process we'll block again in pthread_cond_wait(). */ pthread_mutex_lock(&bio_mutex[type]); listDelNode(bio_jobs[type],ln); bio_pending[type]--; + + /* Unblock threads blocked on bioWaitStepOfType() if any. */ + pthread_cond_broadcast(&bio_step_cond[type]); } } From 51b627d916dc7930879558f7c627dc16022612a8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 13:10:05 +0200 Subject: [PATCH 119/616] Don't perform eviction when re-entering the event loop. Related to #5250. --- src/server.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 55bcb4dc2..fd8472f60 100644 --- a/src/server.c +++ b/src/server.c @@ -2593,8 +2593,13 @@ int processCommand(client *c) { * * First we try to free some memory if possible (if there are volatile * keys in the dataset). If there are not the only thing we can do - * is returning an error. */ - if (server.maxmemory) { + * is returning an error. + * + * Note that we do not want to reclaim memory if we are here re-entering + * the event loop since there is a busy Lua script running in timeout + * condition, to avoid mixing the propagation of scripts with the propagation + * of DELs due to eviction. */ + if (server.maxmemory && !server.lua_timedout) { int out_of_memory = freeMemoryIfNeeded() == C_ERR; /* freeMemoryIfNeeded may flush slave output buffers. This may result * into a slave, that may be the active client, to be freed. */ From 092e4de613af1464d7592cd51e12bd1c87dd0645 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 13:20:12 +0200 Subject: [PATCH 120/616] Propagate read-only scripts as SCRIPT LOAD. See issue #5250 and the new comments added to the code in this commit for details. --- src/scripting.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 8aa62071c..4b5b0002f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1250,6 +1250,7 @@ void evalGenericCommand(client *c, int evalsha) { lua_State *lua = server.lua; char funcname[43]; long long numkeys; + long long initial_server_dirty = server.dirty; int delhook = 0, err; /* When we replicate whole scripts, we want the same PRNG sequence at @@ -1432,9 +1433,21 @@ void evalGenericCommand(client *c, int evalsha) { replicationScriptCacheAdd(c->argv[1]->ptr); serverAssertWithInfo(c,NULL,script != NULL); - rewriteClientCommandArgument(c,0, - resetRefCount(createStringObject("EVAL",4))); - rewriteClientCommandArgument(c,1,script); + + /* If the script did not produce any changes in the dataset we want + * just to replicate it as SCRIPT LOAD, otherwise we risk running + * an aborted script on slaves (that may then produce results there) + * or just running a CPU costly read-only script on the slaves. */ + if (server.dirty == initial_server_dirty) { + rewriteClientCommandVector(c,3, + resetRefCount(createStringObject("SCRIPT",6)), + resetRefCount(createStringObject("LOAD",4)), + script); + } else { + rewriteClientCommandArgument(c,0, + resetRefCount(createStringObject("EVAL",4))); + rewriteClientCommandArgument(c,1,script); + } forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF); } } From 7e11825ef4467ccc64f961761ea050a3df4853ba Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 15:48:08 +0200 Subject: [PATCH 121/616] Safer script stop condition on OOM. Here the idea is that we do not want freeMemoryIfNeeded() to propagate a DEL command before the script and change what happens in the script execution once it reaches the slave. For example see this potential issue (in the words of @soloestoy): On master, we run the following script: if redis.call('get','key') then redis.call('set','xxx','yyy') end redis.call('set','c','d') Then when redis attempts to execute redis.call('set','xxx','yyy'), we call freeMemoryIfNeeded(), and the key may get deleted, and because redis.call('set','xxx','yyy') has already been executed on master, this script will be replicated to slave. But the slave received "DEL key" before the script, and will ignore maxmemory, so after that master has xxx and c, slave has only one key c. Note that this patch (and other related work) was authored collaboratively in issue #5250 with the help of @soloestoy and @oranagra. Related to issue #5250. --- src/scripting.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 4b5b0002f..43f8d09a5 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -512,10 +512,13 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * could enlarge the memory usage are not allowed, but only if this is the * first write in the context of this script, otherwise we can't stop * in the middle. */ - if (server.maxmemory && server.lua_write_dirty == 0 && + if (server.maxmemory && /* Maxmemory is actually enabled. */ + !server.loading && /* Don't care about mem if loading. */ + !server.masterhost && /* Slave must execute the script. */ + server.lua_write_dirty == 0 && /* Script had no side effects so far. */ (cmd->flags & CMD_DENYOOM)) { - if (freeMemoryIfNeeded() == C_ERR) { + if (getMaxmemoryState(NULL,NULL,NULL,NULL) != C_OK) { luaPushError(lua, shared.oomerr->ptr); goto cleanup; } From 7895835df6aeb2a2d499bc499ae6cb340228908f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 19:33:56 +0200 Subject: [PATCH 122/616] Use commands (effects) replication by default in scripts. See issue #5250 and issue #5292 for more info. --- src/config.c | 2 ++ src/scripting.c | 1 - src/server.c | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 1a64d25ed..b4c5b404a 100644 --- a/src/config.c +++ b/src/config.c @@ -677,6 +677,8 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { + server.lua_always_replicate_commands = yesnotoi(argv[1]); } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { diff --git a/src/scripting.c b/src/scripting.c index 43f8d09a5..acea551ea 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -917,7 +917,6 @@ void scriptingInit(int setup) { server.lua_client = NULL; server.lua_caller = NULL; server.lua_timedout = 0; - server.lua_always_replicate_commands = 0; /* Only DEBUG can change it.*/ ldbInit(); } diff --git a/src/server.c b/src/server.c index fd8472f60..af5bef7f9 100644 --- a/src/server.c +++ b/src/server.c @@ -1715,6 +1715,12 @@ void initServerConfig(void) { server.assert_line = 0; server.bug_report_start = 0; server.watchdog_period = 0; + + /* By default we want scripts to be always replicated by effects + * (single commands executed by the script), and not by sending the + * script to the slave / AOF. This is the new way starting from + * Redis 5. However it is possible to revert it via redis.conf. */ + server.lua_always_replicate_commands = 1; } extern char **environ; From 6bd0d342dd566ced07dcad07c35f449b59f6c0cc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 19:47:29 +0200 Subject: [PATCH 123/616] Fix scripting tests now that we default to commands repl. --- tests/unit/scripting.tcl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index bcde721c3..b0591adfa 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -148,9 +148,11 @@ start_server {tags {"scripting"}} { test {EVAL - Scripts can't run certain commands} { set e {} + r debug lua-always-replicate-commands 0 catch { r eval "redis.pcall('randomkey'); return redis.pcall('set','x','ciao')" 0 } e + r debug lua-always-replicate-commands 1 set e } {*not allowed after*} @@ -299,9 +301,12 @@ start_server {tags {"scripting"}} { } {b534286061d4b9e4026607613b95c06c06015ae8 loaded} test "In the context of Lua the output of random commands gets ordered" { + r debug lua-always-replicate-commands 0 r del myset r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz - r eval {return redis.call('smembers',KEYS[1])} 1 myset + set res [r eval {return redis.call('smembers',KEYS[1])} 1 myset] + r debug lua-always-replicate-commands 1 + set res } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} test "SORT is normally not alpha re-ordered for the scripting engine" { @@ -655,11 +660,13 @@ start_server {tags {"scripting repl"}} { } {1} test "Redis.set_repl() must be issued after replicate_commands()" { + r debug lua-always-replicate-commands 0 catch { r eval { redis.set_repl(redis.REPL_ALL); } 0 } e + r debug lua-always-replicate-commands 1 set e } {*only after turning on*} From db74d71eb349465b674ff42af3437969dc594f61 Mon Sep 17 00:00:00 2001 From: Weiliang Li Date: Thu, 6 Sep 2018 13:40:05 +0800 Subject: [PATCH 124/616] fix usage typo in redis-cli --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0e8777bd2..2bb8c56f8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1495,7 +1495,7 @@ static void usage(void) { " --ldb Used with --eval enable the Redis Lua debugger.\n" " --ldb-sync-mode Like --ldb but uses the synchronous Lua debugger, in\n" " this mode the server is blocked and script changes are\n" -" are not rolled back from the server memory.\n" +" not rolled back from the server memory.\n" " --cluster [args...] [opts...]\n" " Cluster Manager command and arguments (see below).\n" " --verbose Verbose mode.\n" From 8d24f8b46b3c544fef33427ca1f2582d92c59dd4 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 6 Sep 2018 20:52:05 +0800 Subject: [PATCH 125/616] Scripting & Streams: some commands need right flags xadd with id * generates random stream id xadd & xtrim with approximate maxlen count may trim stream randomly xinfo may get random radix-tree-keys/nodes xpending may get random idletime xclaim: master and slave may have different idletime in stream --- src/server.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index af5bef7f9..b65d69a16 100644 --- a/src/server.c +++ b/src/server.c @@ -306,7 +306,7 @@ struct redisCommand redisCommandTable[] = { {"pfcount",pfcountCommand,-2,"r",0,NULL,1,-1,1,0,0}, {"pfmerge",pfmergeCommand,-2,"wm",0,NULL,1,-1,1,0,0}, {"pfdebug",pfdebugCommand,-3,"w",0,NULL,0,0,0,0,0}, - {"xadd",xaddCommand,-5,"wmF",0,NULL,1,1,1,0,0}, + {"xadd",xaddCommand,-5,"wmFR",0,NULL,1,1,1,0,0}, {"xrange",xrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xrevrange",xrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xlen",xlenCommand,2,"rF",0,NULL,1,1,1,0,0}, @@ -314,11 +314,11 @@ struct redisCommand redisCommandTable[] = { {"xreadgroup",xreadCommand,-7,"ws",0,xreadGetKeys,1,1,1,0,0}, {"xgroup",xgroupCommand,-2,"wm",0,NULL,2,2,1,0,0}, {"xack",xackCommand,-4,"wF",0,NULL,1,1,1,0,0}, - {"xpending",xpendingCommand,-3,"r",0,NULL,1,1,1,0,0}, - {"xclaim",xclaimCommand,-6,"wF",0,NULL,1,1,1,0,0}, - {"xinfo",xinfoCommand,-2,"r",0,NULL,2,2,1,0,0}, + {"xpending",xpendingCommand,-3,"rR",0,NULL,1,1,1,0,0}, + {"xclaim",xclaimCommand,-6,"wRF",0,NULL,1,1,1,0,0}, + {"xinfo",xinfoCommand,-2,"rR",0,NULL,2,2,1,0,0}, {"xdel",xdelCommand,-3,"wF",0,NULL,1,1,1,0,0}, - {"xtrim",xtrimCommand,-2,"wF",0,NULL,1,1,1,0,0}, + {"xtrim",xtrimCommand,-2,"wFR",0,NULL,1,1,1,0,0}, {"post",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"host:",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"latency",latencyCommand,-2,"aslt",0,NULL,0,0,0,0,0} From ef2c7a5bbb6d62b2f970bbb4a12d149d2d09faaf Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:43:39 +0200 Subject: [PATCH 126/616] Slave removal: SLAVEOF -> REPLICAOF. SLAVEOF is now an alias. --- src/replication.c | 2 +- src/server.c | 3 ++- src/server.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index 6d589c012..11fd34791 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1997,7 +1997,7 @@ void replicationHandleMasterDisconnection(void) { * the slaves only if we'll have to do a full resync with our master. */ } -void slaveofCommand(client *c) { +void replicaofCommand(client *c) { /* SLAVEOF is not allowed in cluster mode as replication is automatically * configured using the current address of the master node. */ if (server.cluster_enabled) { diff --git a/src/server.c b/src/server.c index 9d2a3718d..92a3b17c0 100644 --- a/src/server.c +++ b/src/server.c @@ -260,7 +260,8 @@ struct redisCommand redisCommandTable[] = { {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, {"pttl",pttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"wF",0,NULL,1,1,1,0,0}, - {"slaveof",slaveofCommand,3,"ast",0,NULL,0,0,0,0,0}, + {"slaveof",replicaofCommand,3,"ast",0,NULL,0,0,0,0,0}, + {"replicaof",replicaofCommand,3,"ast",0,NULL,0,0,0,0,0}, {"role",roleCommand,1,"lst",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"last",0,NULL,0,0,0,0,0}, diff --git a/src/server.h b/src/server.h index 09348585b..ae20aff6a 100644 --- a/src/server.h +++ b/src/server.h @@ -1998,7 +1998,7 @@ void ttlCommand(client *c); void touchCommand(client *c); void pttlCommand(client *c); void persistCommand(client *c); -void slaveofCommand(client *c); +void replicaofCommand(client *c); void roleCommand(client *c); void debugCommand(client *c); void msetCommand(client *c); From f579b774490d119df579413e747a1e02ad863462 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:46:28 +0200 Subject: [PATCH 127/616] Slave removal: redis-cli --slave -> --replica. --slave alias remains but is undocumented, just for backward compatibiltiy. --- src/redis-cli.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 536eeb79f..0a3d78db3 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1282,6 +1282,8 @@ static int parseOptions(int argc, char **argv) { config.lru_test_sample_size = strtoll(argv[++i],NULL,10); } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--replica")) { + config.slave_mode = 1; } else if (!strcmp(argv[i],"--stat")) { config.stat_mode = 1; } else if (!strcmp(argv[i],"--scan")) { @@ -1478,7 +1480,7 @@ static void usage(void) { " --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n" " Default time interval is 1 sec. Change it using -i.\n" " --lru-test Simulate a cache workload with an 80-20 distribution.\n" -" --slave Simulate a slave showing commands received from the master.\n" +" --replica Simulate a replica showing commands received from the master.\n" " --rdb Transfer an RDB dump from remote server to local file.\n" " --pipe Transfer raw Redis protocol from stdin to server.\n" " --pipe-timeout In --pipe mode, abort with error if after sending all data.\n" From a9419e23867f10a99927fb3e1689071c60cfe11c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:49:03 +0200 Subject: [PATCH 128/616] Slave removal: Convert cluster.c log messages and command names. --- src/cluster.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 2f3e298e0..421e678bb 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -1230,7 +1230,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) { serverLog(LL_NOTICE, "Clear FAIL state for node %.40s: %s is reachable again.", node->name, - nodeIsSlave(node) ? "slave" : "master without slots"); + nodeIsSlave(node) ? "replica" : "master without slots"); node->flags &= ~CLUSTER_NODE_FAIL; clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); } @@ -2059,7 +2059,7 @@ int clusterProcessPacket(clusterLink *link) { server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT; server.cluster->mf_slave = sender; pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2)); - serverLog(LL_WARNING,"Manual failover requested by slave %.40s.", + serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); } else if (type == CLUSTERMSG_TYPE_UPDATE) { clusterNode *n; /* The node the update is about. */ @@ -2873,7 +2873,7 @@ void clusterLogCantFailover(int reason) { switch(reason) { case CLUSTER_CANT_FAILOVER_DATA_AGE: msg = "Disconnected from master for longer than allowed. " - "Please check the 'cluster-slave-validity-factor' configuration " + "Please check the 'cluster-replica-validity-factor' configuration " "option."; break; case CLUSTER_CANT_FAILOVER_WAITING_DELAY: @@ -3054,7 +3054,7 @@ void clusterHandleSlaveFailover(void) { server.cluster->failover_auth_time += added_delay; server.cluster->failover_auth_rank = newrank; serverLog(LL_WARNING, - "Slave rank updated to #%d, added %lld milliseconds of delay.", + "Replica rank updated to #%d, added %lld milliseconds of delay.", newrank, added_delay); } } @@ -4187,7 +4187,7 @@ void clusterCommand(client *c) { "COUNT-failure-reports -- Return number of failure reports for .", "COUNTKEYSINSLOT - Return the number of keys in .", "DELSLOTS [slot ...] -- Delete slots information from current node.", -"FAILOVER [force|takeover] -- Promote current slave node to being a master.", +"FAILOVER [force|takeover] -- Promote current replica node to being a master.", "FORGET -- Remove a node from the cluster.", "GETKEYSINSLOT -- Return key names stored by current node in a slot.", "FLUSHSLOTS -- Delete current node own slots information.", @@ -4197,11 +4197,11 @@ void clusterCommand(client *c) { "MYID -- Return the node id.", "NODES -- Return cluster configuration seen by node. Output format:", " ... ", -"REPLICATE -- Configure current node as slave to .", +"REPLICATE -- Configure current node as replica to .", "RESET [hard|soft] -- Reset current node (default: soft).", "SET-config-epoch - Set config epoch of current node.", "SETSLOT (importing|migrating|stable|node ) -- Set slot state.", -"SLAVES -- Return slaves.", +"REPLICAS -- Return replicas.", "SLOTS -- Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL @@ -4578,7 +4578,7 @@ NULL /* Can't replicate a slave. */ if (nodeIsSlave(n)) { - addReplyError(c,"I can only replicate a master, not a slave."); + addReplyError(c,"I can only replicate a master, not a replica."); return; } @@ -4597,7 +4597,8 @@ NULL clusterSetMaster(n); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"slaves") && c->argc == 3) { + } else if ((!strcasecmp(c->argv[1]->ptr,"slaves") || + !strcasecmp(c->argv[1]->ptr,"replicas")) && c->argc == 3) { /* CLUSTER SLAVES */ clusterNode *n = clusterLookupNode(c->argv[2]->ptr); int j; @@ -4651,10 +4652,10 @@ NULL /* Check preconditions. */ if (nodeIsMaster(myself)) { - addReplyError(c,"You should send CLUSTER FAILOVER to a slave"); + addReplyError(c,"You should send CLUSTER FAILOVER to a replica"); return; } else if (myself->slaveof == NULL) { - addReplyError(c,"I'm a slave but my master is unknown to me"); + addReplyError(c,"I'm a replica but my master is unknown to me"); return; } else if (!force && (nodeFailed(myself->slaveof) || From 6f5848613956cb2e662bbdc675bbaaf7be42574a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:22:20 +0200 Subject: [PATCH 129/616] Slave removal: config.c converted + config rewriting hacks. Aliases added for all the commands mentioning slave. Moreover CONFIG REWRITE will use the new names, and will be able to reuse the old lines mentioning the old options. --- src/config.c | 155 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 38 deletions(-) diff --git a/src/config.c b/src/config.c index b4c5b404a..3b38f98d8 100644 --- a/src/config.c +++ b/src/config.c @@ -344,15 +344,19 @@ void loadServerConfigFromString(char *config) { err = "lfu-decay-time must be 0 or greater"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slaveof") && argc == 3) { + } else if ((!strcasecmp(argv[0],"slaveof") || + !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; server.masterhost = sdsnew(argv[1]); server.masterport = atoi(argv[2]); server.repl_state = REPL_STATE_CONNECT; - } else if (!strcasecmp(argv[0],"repl-ping-slave-period") && argc == 2) { + } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || + !strcasecmp(argv[0],"repl-ping-replica-period")) && + argc == 2) + { server.repl_ping_slave_period = atoi(argv[1]); if (server.repl_ping_slave_period <= 0) { - err = "repl-ping-slave-period must be 1 or greater"; + err = "repl-ping-replica-period must be 1 or greater"; goto loaderr; } } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { @@ -391,15 +395,24 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { zfree(server.masterauth); server.masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"slave-serve-stale-data") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-serve-stale-data") || + !strcasecmp(argv[0],"replica-serve-stale-data")) + && argc == 2) + { if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-read-only") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-read-only") || + !strcasecmp(argv[0],"replica-read-only")) + && argc == 2) + { if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-ignore-maxmemory") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-ignore-maxmemory") || + !strcasecmp(argv[0],"replica-ignore-maxmemory")) + && argc == 2) + { if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } @@ -427,7 +440,9 @@ void loadServerConfigFromString(char *config) { if ((server.lazyfree_lazy_server_del = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-lazy-flush") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-lazy-flush") || + !strcasecmp(argv[0],"replica-lazy-flush")) && argc == 2) + { if ((server.repl_slave_lazy_flush = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } @@ -659,15 +674,17 @@ void loadServerConfigFromString(char *config) { err = "cluster migration barrier must zero or positive"; goto loaderr; } - } else if (!strcasecmp(argv[0],"cluster-slave-validity-factor") + } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || + !strcasecmp(argv[0],"cluster-replica-validity-factor")) && argc == 2) { server.cluster_slave_validity_factor = atoi(argv[1]); if (server.cluster_slave_validity_factor < 0) { - err = "cluster slave validity factor must be zero or positive"; + err = "cluster replica validity factor must be zero or positive"; goto loaderr; } - } else if (!strcasecmp(argv[0],"cluster-slave-no-failover") && + } else if ((!strcasecmp(argv[0],"cluster-slave-no-failover") || + !strcasecmp(argv[0],"cluster-replica-no-failiver")) && argc == 2) { server.cluster_slave_no_failover = yesnotoi(argv[1]); @@ -720,27 +737,37 @@ void loadServerConfigFromString(char *config) { if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-priority") || + !strcasecmp(argv[0],"replica-priority")) && argc == 2) + { server.slave_priority = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"slave-announce-ip") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-announce-ip") || + !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) + { zfree(server.slave_announce_ip); server.slave_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"slave-announce-port") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-announce-port") || + !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) + { server.slave_announce_port = atoi(argv[1]); if (server.slave_announce_port < 0 || server.slave_announce_port > 65535) { err = "Invalid port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"min-slaves-to-write") && argc == 2) { + } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || + !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) + { server.repl_min_slaves_to_write = atoi(argv[1]); if (server.repl_min_slaves_to_write < 0) { - err = "Invalid value for min-slaves-to-write."; goto loaderr; + err = "Invalid value for min-replicas-to-write."; goto loaderr; } - } else if (!strcasecmp(argv[0],"min-slaves-max-lag") && argc == 2) { + } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || + !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) + { server.repl_min_slaves_max_lag = atoi(argv[1]); if (server.repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-slaves-max-lag."; goto loaderr; + err = "Invalid value for min-replicas-max-lag."; goto loaderr; } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -782,7 +809,7 @@ void loadServerConfigFromString(char *config) { if (server.cluster_enabled && server.masterhost) { linenum = slaveof_linenum; i = linenum-1; - err = "slaveof directive not allowed in cluster mode"; + err = "replicaof directive not allowed in cluster mode"; goto loaderr; } @@ -866,6 +893,10 @@ void loadServerConfig(char *filename, char *options) { #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { +#define config_set_special_field_with_alias(_name1,_name2) \ + } else if (!strcasecmp(c->argv[2]->ptr,_name1) || \ + !strcasecmp(c->argv[2]->ptr,_name2)) { + #define config_set_else } else void configSetCommand(client *c) { @@ -1025,7 +1056,9 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } config_set_special_field("slave-announce-ip") { + } config_set_special_field_with_alias("slave-announce-ip", + "replica-announce-ip") + { zfree(server.slave_announce_ip); server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; @@ -1041,6 +1074,8 @@ void configSetCommand(client *c) { "cluster-require-full-coverage",server.cluster_require_full_coverage) { } config_set_bool_field( "cluster-slave-no-failover",server.cluster_slave_no_failover) { + } config_set_bool_field( + "cluster-replica-no-failover",server.cluster_slave_no_failover) { } config_set_bool_field( "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { } config_set_bool_field( @@ -1051,10 +1086,16 @@ void configSetCommand(client *c) { "aof-use-rdb-preamble",server.aof_use_rdb_preamble) { } config_set_bool_field( "slave-serve-stale-data",server.repl_serve_stale_data) { + } config_set_bool_field( + "replica-serve-stale-data",server.repl_serve_stale_data) { } config_set_bool_field( "slave-read-only",server.repl_slave_ro) { + } config_set_bool_field( + "replica-read-only",server.repl_slave_ro) { } config_set_bool_field( "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { + } config_set_bool_field( + "replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { } config_set_bool_field( "activerehashing",server.activerehashing) { } config_set_bool_field( @@ -1082,6 +1123,8 @@ void configSetCommand(client *c) { "lazyfree-lazy-server-del",server.lazyfree_lazy_server_del) { } config_set_bool_field( "slave-lazy-flush",server.repl_slave_lazy_flush) { + } config_set_bool_field( + "replica-lazy-flush",server.repl_slave_lazy_flush) { } config_set_bool_field( "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { } config_set_bool_field( @@ -1145,6 +1188,8 @@ void configSetCommand(client *c) { "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ } config_set_numerical_field( "repl-ping-slave-period",server.repl_ping_slave_period,1,INT_MAX) { + } config_set_numerical_field( + "repl-ping-replica-period",server.repl_ping_slave_period,1,INT_MAX) { } config_set_numerical_field( "repl-timeout",server.repl_timeout,1,INT_MAX) { } config_set_numerical_field( @@ -1171,6 +1216,8 @@ void configSetCommand(client *c) { "cluster-migration-barrier",server.cluster_migration_barrier,0,INT_MAX){ } config_set_numerical_field( "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { + } config_set_numerical_field( + "cluster-replica-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { } config_set_numerical_field( "hz",server.config_hz,0,INT_MAX) { /* Hz is more an hint from the user, so we accept values out of range @@ -1282,6 +1329,7 @@ void configGetCommand(client *c) { config_get_string_field("logfile",server.logfile); config_get_string_field("pidfile",server.pidfile); config_get_string_field("slave-announce-ip",server.slave_announce_ip); + config_get_string_field("replica-announce-ip",server.slave_announce_ip); /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); @@ -1334,19 +1382,25 @@ void configGetCommand(client *c) { config_get_numerical_field("tcp-backlog",server.tcp_backlog); config_get_numerical_field("databases",server.dbnum); config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); + config_get_numerical_field("repl-ping-replica-period",server.repl_ping_slave_period); config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); config_get_numerical_field("slave-priority",server.slave_priority); + config_get_numerical_field("replica-priority",server.slave_priority); config_get_numerical_field("slave-announce-port",server.slave_announce_port); + config_get_numerical_field("replica-announce-port",server.slave_announce_port); config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); + config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); + config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); config_get_numerical_field("hz",server.config_hz); config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); + config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); @@ -1355,14 +1409,22 @@ void configGetCommand(client *c) { server.cluster_require_full_coverage); config_get_bool_field("cluster-slave-no-failover", server.cluster_slave_no_failover); + config_get_bool_field("cluster-replica-no-failover", + server.cluster_slave_no_failover); config_get_bool_field("no-appendfsync-on-rewrite", server.aof_no_fsync_on_rewrite); config_get_bool_field("slave-serve-stale-data", server.repl_serve_stale_data); + config_get_bool_field("replica-serve-stale-data", + server.repl_serve_stale_data); config_get_bool_field("slave-read-only", server.repl_slave_ro); + config_get_bool_field("replica-read-only", + server.repl_slave_ro); config_get_bool_field("slave-ignore-maxmemory", server.repl_slave_ignore_maxmemory); + config_get_bool_field("replica-ignore-maxmemory", + server.repl_slave_ignore_maxmemory); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); @@ -1391,6 +1453,8 @@ void configGetCommand(client *c) { server.lazyfree_lazy_server_del); config_get_bool_field("slave-lazy-flush", server.repl_slave_lazy_flush); + config_get_bool_field("replica-lazy-flush", + server.repl_slave_lazy_flush); config_get_bool_field("dynamic-hz", server.dynamic_hz); @@ -1464,10 +1528,14 @@ void configGetCommand(client *c) { addReplyBulkCString(c,buf); matches++; } - if (stringmatch(pattern,"slaveof",1)) { + if (stringmatch(pattern,"slaveof",1) || + stringmatch(pattern,"replicaof",1)) + { + char *optname = stringmatch(pattern,"slaveof",1) ? + "slaveof" : "replicaof"; char buf[256]; - addReplyBulkCString(c,"slaveof"); + addReplyBulkCString(c,optname); if (server.masterhost) snprintf(buf,sizeof(buf),"%s %d", server.masterhost, server.masterport); @@ -1623,8 +1691,20 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { /* Now we populate the state according to the content of this line. * Append the line and populate the option -> line numbers map. */ rewriteConfigAppendLine(state,line); - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* Translate options using the word "slave" to the corresponding name + * "replica", before adding such option to the config name -> lines + * mapping. */ + char *p = strstr(argv[0],"slave"); + if (p) { + sds alt = sdsempty(); + alt = sdscatlen(alt,argv[0],p-argv[0]);; + alt = sdscatlen(alt,"replica",7); + alt = sdscatlen(alt,p+5,strlen(p+5)); + sdsfree(argv[0]); + argv[0] = alt; + } + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); sdsfreesplitres(argv,argc); } fclose(fp); @@ -1811,15 +1891,14 @@ void rewriteConfigDirOption(struct rewriteConfigState *state) { } /* Rewrite the slaveof option. */ -void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { - char *option = "slaveof"; +void rewriteConfigSlaveofOption(struct rewriteConfigState *state, char *option) { sds line; /* If this is a master, we want all the slaveof config options * in the file to be removed. Note that if this is a cluster instance * we don't want a slaveof directive inside redis.conf. */ if (server.cluster_enabled || server.masterhost == NULL) { - rewriteConfigMarkAsProcessed(state,"slaveof"); + rewriteConfigMarkAsProcessed(state,option); return; } line = sdscatprintf(sdsempty(),"%s %s %d", option, @@ -2040,7 +2119,7 @@ int rewriteConfig(char *path) { rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"slave-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); + rewriteConfigNumericalOption(state,"replica-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); @@ -2053,23 +2132,23 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,CONFIG_DEFAULT_RDB_CHECKSUM); rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); - rewriteConfigSlaveofOption(state); - rewriteConfigStringOption(state,"slave-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); + rewriteConfigSlaveofOption(state,"replicaof"); + rewriteConfigStringOption(state,"replica-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); - rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); - rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); - rewriteConfigYesNoOption(state,"slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); - rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); + rewriteConfigYesNoOption(state,"replica-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); + rewriteConfigYesNoOption(state,"replica-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); + rewriteConfigYesNoOption(state,"replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); + rewriteConfigNumericalOption(state,"repl-ping-replica-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigYesNoOption(state,"repl-diskless-sync",server.repl_diskless_sync,CONFIG_DEFAULT_REPL_DISKLESS_SYNC); rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-slaves-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-slaves-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); + rewriteConfigNumericalOption(state,"replica-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); + rewriteConfigNumericalOption(state,"min-replicas-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); + rewriteConfigNumericalOption(state,"min-replicas-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); @@ -2095,10 +2174,10 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigYesNoOption(state,"cluster-require-full-coverage",server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE); - rewriteConfigYesNoOption(state,"cluster-slave-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); + rewriteConfigYesNoOption(state,"cluster-replica-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-slave-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); + rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); @@ -2126,7 +2205,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION); rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); - rewriteConfigYesNoOption(state,"slave-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); + rewriteConfigYesNoOption(state,"replica-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); /* Rewrite Sentinel config if in Sentinel mode. */ From 6f3d357d8feca56b524d680360faf99d3801920c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:27:37 +0200 Subject: [PATCH 130/616] Slave removal: slave -> replica in redis.conf and output buffer option. --- redis.conf | 256 +++++++++++++++++++++++------------------------ src/config.c | 4 +- src/networking.c | 1 + 3 files changed, 132 insertions(+), 129 deletions(-) diff --git a/redis.conf b/redis.conf index 886c64f21..868f14c48 100644 --- a/redis.conf +++ b/redis.conf @@ -264,59 +264,59 @@ dir ./ ################################# REPLICATION ################################# -# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# Master-Replica replication. Use replcaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # # 1) Redis replication is asynchronous, but you can configure a master to # stop accepting writes if it appears to be not connected with at least -# a given number of slaves. -# 2) Redis slaves are able to perform a partial resynchronization with the +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the # master if the replication link is lost for a relatively small amount of # time. You may want to configure the replication backlog size (see the next # sections of this file) with a sensible value depending on your needs. # 3) Replication is automatic and does not need user intervention. After a -# network partition slaves automatically try to reconnect to masters +# network partition replicas automatically try to reconnect to masters # and resynchronize with them. # -# slaveof +# replicaof # If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before +# directive below) it is possible to tell the replica to authenticate before # starting the replication synchronization process, otherwise the master will -# refuse the slave request. +# refuse the replica request. # # masterauth -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: # -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# 2) if replica-serve-stale-data is set to 'no' the replica will reply with # an error "SYNC with master in progress" to all the kind of commands -# but to INFO, SLAVEOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, +# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, # SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, # COMMAND, POST, HOST: and LATENCY. # -slave-serve-stale-data yes +replica-serve-stale-data yes -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but # may also cause problems if clients are writing to it because of a # misconfiguration. # -# Since Redis 2.6 by default slaves are read-only. +# Since Redis 2.6 by default replicas are read-only. # -# Note: read only slaves are not designed to be exposed to untrusted clients +# Note: read only replicas are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. -# Still a read only slave exports by default all the administrative commands +# Still a read only replica exports by default all the administrative commands # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only slaves using 'rename-command' to shadow all the +# security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. -slave-read-only yes +replica-read-only yes # Replication SYNC strategy: disk or socket. # @@ -324,25 +324,25 @@ slave-read-only yes # WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY # ------------------------------------------------------- # -# New slaves and reconnecting slaves that are not able to continue the replication +# New replicas and reconnecting replicas that are not able to continue the replication # process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the slaves. +# synchronization". An RDB file is transmitted from the master to the replicas. # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB # file on disk. Later the file is transferred by the parent -# process to the slaves incrementally. +# process to the replicas incrementally. # 2) Diskless: The Redis master creates a new process that directly writes the -# RDB file to slave sockets, without touching the disk at all. +# RDB file to replica sockets, without touching the disk at all. # -# With disk-backed replication, while the RDB file is generated, more slaves +# With disk-backed replication, while the RDB file is generated, more replicas # can be queued and served with the RDB file as soon as the current child producing # the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new slaves arriving will be queued and a new transfer +# the transfer starts, new replicas arriving will be queued and a new transfer # will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple slaves +# time (in seconds) before starting the transfer in the hope that multiple replicas # will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication @@ -351,140 +351,140 @@ repl-diskless-sync no # When diskless replication is enabled, it is possible to configure the delay # the server waits in order to spawn the child that transfers the RDB via socket -# to the slaves. +# to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new slaves arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more slaves arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 +# replicas send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_replica_period option. The default value is 10 # seconds. # -# repl-ping-slave-period 10 +# repl-ping-replica-period 10 # The following option sets the replication timeout for: # -# 1) Bulk transfer I/O during SYNC, from the point of view of slave. -# 2) Master timeout from the point of view of slaves (data, pings). -# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) replica timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. # # repl-timeout 60 -# Disable TCP_NODELAY on the slave socket after SYNC? +# Disable TCP_NODELAY on the replica socket after SYNC? # # If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to slaves. But this can add a delay for -# the data to appear on the slave side, up to 40 milliseconds with +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with # Linux kernels using a default configuration. # -# If you select "no" the delay for data to appear on the slave side will +# If you select "no" the delay for data to appear on the replica side will # be reduced but more bandwidth will be used for replication. # # By default we optimize for low latency, but in very high traffic conditions -# or when the master and slaves are many hops away, turning this to "yes" may +# or when the master and replicas are many hops away, turning this to "yes" may # be a good idea. repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# slave data when slaves are disconnected for some time, so that when a slave +# replica data when replicas are disconnected for some time, so that when a replica # wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the slave missed while +# resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The bigger the replication backlog, the longer the time the slave can be +# The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a slave connected. +# The backlog is only allocated once there is at least a replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected slaves for some time, the backlog +# After a master has no longer connected replicas for some time, the backlog # will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last slave disconnected, for +# need to elapse, starting from the time the last replica disconnected, for # the backlog buffer to be freed. # -# Note that slaves never free the backlog for timeout, since they may be +# Note that replicas never free the backlog for timeout, since they may be # promoted to masters later, and should be able to correctly "partially -# resynchronize" with the slaves: hence they should always accumulate backlog. +# resynchronize" with the replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The slave priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a +# The replica priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a replica to promote into a # master if the master is no longer working correctly. # -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel will # pick the one with priority 10, that is the lowest. # -# However a special priority of 0 marks the slave as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by # Redis Sentinel for promotion. # # By default the priority is 100. -slave-priority 100 +replica-priority 100 # It is possible for a master to stop accepting writes if there are less than -# N slaves connected, having a lag less or equal than M seconds. +# N replicas connected, having a lag less or equal than M seconds. # -# The N slaves need to be in "online" state. +# The N replicas need to be in "online" state. # # The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the slave, that is usually sent every second. +# the last ping received from the replica, that is usually sent every second. # # This option does not GUARANTEE that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough slaves +# will limit the window of exposure for lost writes in case not enough replicas # are available, to the specified number of seconds. # -# For example to require at least 3 slaves with a lag <= 10 seconds use: +# For example to require at least 3 replicas with a lag <= 10 seconds use: # -# min-slaves-to-write 3 -# min-slaves-max-lag 10 +# min-replicas-to-write 3 +# min-replicas-max-lag 10 # # Setting one or the other to 0 disables the feature. # -# By default min-slaves-to-write is set to 0 (feature disabled) and -# min-slaves-max-lag is set to 10. +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. # A Redis master is able to list the address and port of the attached -# slaves in different ways. For example the "INFO replication" section +# replicas in different ways. For example the "INFO replication" section # offers this information, which is used, among other tools, by -# Redis Sentinel in order to discover slave instances. +# Redis Sentinel in order to discover replica instances. # Another place where this info is available is in the output of the # "ROLE" command of a master. # -# The listed IP and address normally reported by a slave is obtained +# The listed IP and address normally reported by a replica is obtained # in the following way: # # IP: The address is auto detected by checking the peer address -# of the socket used by the slave to connect with the master. +# of the socket used by the replica to connect with the master. # -# Port: The port is communicated by the slave during the replication -# handshake, and is normally the port that the slave is using to +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to # list for connections. # # However when port forwarding or Network Address Translation (NAT) is -# used, the slave may be actually reachable via different IP and port -# pairs. The following two options can be used by a slave in order to +# used, the replica may be actually reachable via different IP and port +# pairs. The following two options can be used by a replica in order to # report to its master a specific set of IP and port, so that both INFO # and ROLE will report those values. # # There is no need to use both the options if you need to override just # the port or the IP address. # -# slave-announce-ip 5.5.5.5 -# slave-announce-port 1234 +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 ################################## SECURITY ################################### @@ -518,7 +518,7 @@ slave-priority 100 # rename-command CONFIG "" # # Please note that changing the name of commands that are logged into the -# AOF file or transmitted to slaves may cause problems. +# AOF file or transmitted to replicas may cause problems. ################################### CLIENTS #################################### @@ -547,15 +547,15 @@ slave-priority 100 # This option is usually useful when using Redis as an LRU or LFU cache, or to # set a hard memory limit for an instance (using the 'noeviction' policy). # -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted # from the used memory count, so that network problems / resyncs will # not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion +# buffer of replicas is full with DELs of keys evicted triggering the deletion # of more keys, and so forth until the database is completely emptied. # -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # # maxmemory @@ -602,25 +602,25 @@ slave-priority 100 # # maxmemory-samples 5 -# Starting from Redis 5, by default a slave will ignore its maxmemory setting +# Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means # that the eviction of keys will be just handled by the master, sending the -# DEL commands to the slave as keys evict in the master side. +# DEL commands to the replica as keys evict in the master side. # -# This behavior ensures that masters and slaves stay consistent, and is usually -# what you want, however if your slave is writable, or you want the slave to have +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica to have # a different memory setting, and you are sure all the writes performed to the -# slave are idempotent, then you may change this default (but be sure to understand +# replica are idempotent, then you may change this default (but be sure to understand # what you are doing). # -# Note that since the slave by default does not evict, it may end using more +# Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the slave, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your slaves and make sure they have enough +# be larger on the replica, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your replicas and make sure they have enough # memory to never hit a real out-of-memory condition before the master hits # the configured maxmemory setting. # -# slave-ingore-maxmemory yes +# replica-ingore-maxmemory yes ############################# LAZY FREEING #################################### @@ -657,7 +657,7 @@ slave-priority 100 # or SORT with STORE option may delete existing keys. The SET command # itself removes any old content of the specified key in order to replace # it with the specified string. -# 4) During replication, when a slave performs a full resynchronization with +# 4) During replication, when a replica performs a full resynchronization with # its master, the content of the whole database is removed in order to # load the RDB file just transferred. # @@ -669,7 +669,7 @@ slave-priority 100 lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no -slave-lazy-flush no +replica-lazy-flush no ############################## APPEND ONLY MODE ############################### @@ -846,42 +846,42 @@ lua-time-limit 5000 # # cluster-node-timeout 15000 -# A slave of a failing master will avoid to start a failover if its data +# A replica of a failing master will avoid to start a failover if its data # looks too old. # -# There is no simple way for a slave to actually have an exact measure of +# There is no simple way for a replica to actually have an exact measure of # its "data age", so the following two checks are performed: # -# 1) If there are multiple slaves able to failover, they exchange messages -# in order to try to give an advantage to the slave with the best +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best # replication offset (more data from the master processed). -# Slaves will try to get their rank by offset, and apply to the start +# replicas will try to get their rank by offset, and apply to the start # of the failover a delay proportional to their rank. # -# 2) Every single slave computes the time of the last interaction with +# 2) Every single replica computes the time of the last interaction with # its master. This can be the last ping or command received (if the master # is still in the "connected" state), or the time that elapsed since the # disconnection with the master (if the replication link is currently down). -# If the last interaction is too old, the slave will not try to failover +# If the last interaction is too old, the replica will not try to failover # at all. # -# The point "2" can be tuned by user. Specifically a slave will not perform +# The point "2" can be tuned by user. Specifically a replica will not perform # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * slave-validity-factor) + repl-ping-slave-period +# (node-timeout * replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the slave-validity-factor -# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the -# slave will not try to failover if it was not able to talk with the master +# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large slave-validity-factor may allow slaves with too old data to failover +# A large replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to -# elect a slave at all. +# elect a replica at all. # -# For maximum availability, it is possible to set the slave-validity-factor -# to a value of 0, which means, that slaves will always try to failover the +# For maximum availability, it is possible to set the replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their # offset rank). @@ -889,22 +889,22 @@ lua-time-limit 5000 # Zero is the only value able to guarantee that when all the partitions heal # the cluster will always be able to continue. # -# cluster-slave-validity-factor 10 +# cluster-replica-validity-factor 10 -# Cluster slaves are able to migrate to orphaned masters, that are masters -# that are left without working slaves. This improves the cluster ability +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability # to resist to failures as otherwise an orphaned master can't be failed over -# in case of failure if it has no working slaves. +# in case of failure if it has no working replicas. # -# Slaves migrate to orphaned masters only if there are still at least a -# given number of other working slaves for their old master. This number -# is the "migration barrier". A migration barrier of 1 means that a slave -# will migrate only if there is at least 1 other working slave for its master -# and so forth. It usually reflects the number of slaves you want for every +# replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every # master in your cluster. # -# Default is 1 (slaves migrate only if their masters remain with at least -# one slave). To disable migration just set it to a very large value. +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # @@ -923,7 +923,7 @@ lua-time-limit 5000 # # cluster-require-full-coverage yes -# This option, when set to yes, prevents slaves from trying to failover its +# This option, when set to yes, prevents replicas from trying to failover its # master during master failures. However the master can still perform a # manual failover, if forced to do so. # @@ -931,7 +931,7 @@ lua-time-limit 5000 # data center operations, where we want one side to never be promoted if not # in the case of a total DC failure. # -# cluster-slave-no-failover no +# cluster-replica-no-failover no # In order to setup your cluster make sure to read the documentation # available at http://redis.io web site. @@ -1165,7 +1165,7 @@ activerehashing yes # The limit can be set differently for the three different classes of clients: # # normal -> normal clients including MONITOR clients -# slave -> slave clients +# replica -> replica clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: @@ -1186,12 +1186,12 @@ activerehashing yes # asynchronous clients may create a scenario where data is requested faster # than it can read. # -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. # # Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 # Client query buffers accumulate new commands. They are limited to a fixed diff --git a/src/config.c b/src/config.c index 3b38f98d8..2feb39336 100644 --- a/src/config.c +++ b/src/config.c @@ -1940,8 +1940,10 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state rewriteConfigFormatMemory(soft,sizeof(soft), server.client_obuf_limits[j].soft_limit_bytes); + char *typename = getClientTypeName(j); + if (!strcmp(typename,"slave")) typename = "replica"; line = sdscatprintf(sdsempty(),"%s %s %s %s %ld", - option, getClientTypeName(j), hard, soft, + option, typename, hard, soft, (long) server.client_obuf_limits[j].soft_limit_seconds); rewriteConfigRewriteLine(state,option,line,force); } diff --git a/src/networking.c b/src/networking.c index 01307c7a3..df670b7c7 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1991,6 +1991,7 @@ int getClientType(client *c) { int getClientTypeByName(char *name) { if (!strcasecmp(name,"normal")) return CLIENT_TYPE_NORMAL; else if (!strcasecmp(name,"slave")) return CLIENT_TYPE_SLAVE; + else if (!strcasecmp(name,"replica")) return CLIENT_TYPE_SLAVE; else if (!strcasecmp(name,"pubsub")) return CLIENT_TYPE_PUBSUB; else if (!strcasecmp(name,"master")) return CLIENT_TYPE_MASTER; else return -1; From a67181f88ae7cb42b3b2978ff3b15fcd6706a7cb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:56:02 +0200 Subject: [PATCH 131/616] Slave removal: fix typo of replicaof. --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 868f14c48..66ad3418a 100644 --- a/redis.conf +++ b/redis.conf @@ -264,7 +264,7 @@ dir ./ ################################# REPLICATION ################################# -# Master-Replica replication. Use replcaof to make a Redis instance a copy of +# Master-Replica replication. Use replicaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # # 1) Redis replication is asynchronous, but you can configure a master to From 8296e5c8464e406af946edb5d520f4c0c2b67997 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:37:23 +0200 Subject: [PATCH 132/616] Slave removal: slave mode -> replica mode text in redis-cli. --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0a3d78db3..c9e8d26b3 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1154,7 +1154,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) { } if (config.slave_mode) { - printf("Entering slave output mode... (press Ctrl-C to quit)\n"); + printf("Entering replica output mode... (press Ctrl-C to quit)\n"); slaveMode(); config.slave_mode = 0; zfree(argvlen); From c9bab6f30231e0ba19c03970073921f17894a83b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:39:51 +0200 Subject: [PATCH 133/616] Slave removal: Make obvious in redis.conf what a replica is. --- redis.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/redis.conf b/redis.conf index 66ad3418a..3ca9fe5bc 100644 --- a/redis.conf +++ b/redis.conf @@ -267,6 +267,11 @@ dir ./ # Master-Replica replication. Use replicaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# # 1) Redis replication is asynchronous, but you can configure a master to # stop accepting writes if it appears to be not connected with at least # a given number of replicas. From 05e8db24ed22c14d26959a413161dd1092eea706 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:46:14 +0200 Subject: [PATCH 134/616] Slave removal: blocked.c logs fixed. --- src/blocked.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index 00212ed69..2b43f2b75 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -215,7 +215,7 @@ void disconnectAllBlockedClients(void) { if (c->flags & CLIENT_BLOCKED) { addReplySds(c,sdsnew( "-UNBLOCKED force unblock from blocking operation, " - "instance state changed (master -> slave?)\r\n")); + "instance state changed (master -> replica?)\r\n")); unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } From cff5f36d94aa34df6a9b62d0bc71bdb8b38116fb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 17:02:44 +0200 Subject: [PATCH 135/616] Slave removal: networking.c logs fixed. --- src/networking.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index df670b7c7..06715ee41 100644 --- a/src/networking.c +++ b/src/networking.c @@ -355,8 +355,8 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE)) { - char* to = c->flags & CLIENT_MASTER? "master": "slave"; - char* from = c->flags & CLIENT_MASTER? "slave": "master"; + char* to = c->flags & CLIENT_MASTER? "master": "replica"; + char* from = c->flags & CLIENT_MASTER? "replica": "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " @@ -836,7 +836,7 @@ void freeClient(client *c) { /* Log link disconnection with slave */ if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { - serverLog(LL_WARNING,"Connection with slave %s lost.", + serverLog(LL_WARNING,"Connection with replica %s lost.", replicationGetSlaveName(c)); } @@ -1654,10 +1654,10 @@ void clientCommand(client *c) { "kill -- Kill connection made from .", "kill