From 2b7ab08e46982bd3db82f558037437d0cc1b1d73 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 16 Jan 2015 09:03:00 +0000 Subject: [PATCH 001/616] Make some defaults explicit in the sentinel.conf for package maintainers This may look a little pointless (and it is a complete no-op change here) but as package maintainers need to modify these lines to actually daemonize (etc. etc) but it's far preferable if the diff is restricted to actually changing just that bit, not adding docs, etc. The less diff the better, in general. Signed-off-by: Chris Lamb --- sentinel.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sentinel.conf b/sentinel.conf index 39d1044e2..d627b8536 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -4,6 +4,31 @@ # The port that this sentinel instance will run on port 26379 +# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it. +# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when +# daemonized. +daemonize no + +# When running daemonized, Redis Sentinel writes a pid file in +# /var/run/redis-sentinel.pid by default. You can specify a custom pid file +# location here. +pidfile /var/run/redis-sentinel.pid + +# By default Redis Sentinel listens for connections from all the network +# interfaces available on the server. It is possible to listen to just one or +# multiple interfaces using the "bind" configuration directive, followed by one +# or more IP addresses. +# +# Examples: +# +# bind 192.168.1.100 10.0.0.1 +# bind 127.0.0.1 + +# Specify the log file name. Also the empty string can be used to force +# Sentinel to log on the standard output. Note that if you use standard +# output for logging but daemonize, logs will be sent to /dev/null +logfile "" + # sentinel announce-ip # sentinel announce-port # From 5494a90bdbc976bd3c503403f49f7049366effed Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Wed, 4 Feb 2015 18:29:22 +0000 Subject: [PATCH 002/616] Tidy grammar in CONFIG SET maxmemory warning. Signed-off-by: Chris Lamb --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 8255a56b7..f0a019517 100644 --- a/src/config.c +++ b/src/config.c @@ -648,7 +648,7 @@ void configSetCommand(redisClient *c) { server.maxmemory = ll; if (server.maxmemory) { if (server.maxmemory < zmalloc_used_memory()) { - redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); + redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); } freeMemoryIfNeeded(); } From 0a86e5e6a5bcce6919acc85b1d1d4bf32adf37bb Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Fri, 29 Apr 2016 16:45:53 +0100 Subject: [PATCH 003/616] Use SOURCE_DATE_EPOCH over unreproducible uname + date calls. See for more details. Signed-off-by: Chris Lamb --- src/mkreleasehdr.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index 1ae95886b..e6d558b17 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -2,6 +2,9 @@ GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` +if [ -n "$SOURCE_DATE_EPOCH" ]; then + BUILD_ID=$(date -u -d "@$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u -r "$SOURCE_DATE_EPOCH" +%s 2>/dev/null || date -u %s) +fi test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ (cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already up-to-date From 17c8ff5a1cd2940070cf64d44d771b1183e96bed Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sun, 10 Dec 2017 17:54:56 +0200 Subject: [PATCH 004/616] Standardizes `MEMORY HELP` subcommand --- src/object.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/object.c b/src/object.c index 43ab6b5f0..52d2d4662 100644 --- a/src/object.c +++ b/src/object.c @@ -1120,7 +1120,18 @@ NULL void memoryCommand(client *c) { robj *o; - if (!strcasecmp(c->argv[1]->ptr,"usage") && c->argc >= 3) { + if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { + + const char *help[] = { +"doctor - Return memory problems reports.", +"malloc-stats -- Return internal statistics report from the memory allocator.", +"purge -- Attempt to purge dirty pages for reclamation by the allocator.", +"stats -- Return information about the memory usage of the server.", +"usage [samples ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", +NULL + }; + addReplyHelp(c, help); + } else if (!strcasecmp(c->argv[1]->ptr,"usage") && c->argc >= 3) { long long samples = OBJ_COMPUTE_SIZE_DEF_SAMPLES; for (int j = 3; j < c->argc; j++) { if (!strcasecmp(c->argv[j]->ptr,"samples") && @@ -1234,19 +1245,7 @@ void memoryCommand(client *c) { addReply(c, shared.ok); /* Nothing to do for other allocators. */ #endif - } else if (!strcasecmp(c->argv[1]->ptr,"help") && c->argc == 2) { - addReplyMultiBulkLen(c,5); - addReplyBulkCString(c, -"MEMORY DOCTOR - Outputs memory problems report"); - addReplyBulkCString(c, -"MEMORY USAGE [SAMPLES ] - Estimate memory usage of key"); - addReplyBulkCString(c, -"MEMORY STATS - Show memory usage details"); - addReplyBulkCString(c, -"MEMORY PURGE - Ask the allocator to release memory"); - addReplyBulkCString(c, -"MEMORY MALLOC-STATS - Show allocator internal stats"); } else { - addReplyError(c,"Syntax error. Try MEMORY HELP"); + addReplyErrorFormat(c, "Unknown subcommand or wrong number of arguments for '%s'. Try MEMORY HELP", (char*)c->argv[1]->ptr); } } From 3c860af35335b4c2304bd2d31ab7230d85ab7ce7 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 15 Dec 2017 21:19:41 +0200 Subject: [PATCH 005/616] Uppercases subcommands in MEMORY HELP --- src/object.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/object.c b/src/object.c index 52d2d4662..606cdd7e5 100644 --- a/src/object.c +++ b/src/object.c @@ -1123,11 +1123,11 @@ void memoryCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"doctor - Return memory problems reports.", -"malloc-stats -- Return internal statistics report from the memory allocator.", -"purge -- Attempt to purge dirty pages for reclamation by the allocator.", -"stats -- Return information about the memory usage of the server.", -"usage [samples ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", +"DOCTOR - Return memory problems reports.", +"MALLOC-STATS -- Return internal statistics report from the memory allocator.", +"PURGE -- Attempt to purge dirty pages for reclamation by the allocator.", +"STATS -- Return information about the memory usage of the server.", +"USAGE [SAMPLES ] -- Return memory in bytes used by and its value. Nested values are sampled up to times (default: 5).", NULL }; addReplyHelp(c, help); From 8f4aff385cdc954f3ee8f0d2af217590401a1b9c Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Fri, 15 Dec 2017 21:21:12 +0200 Subject: [PATCH 006/616] Uppercases subcommands in OBJECT HELP --- src/object.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index 606cdd7e5..b979b458a 100644 --- a/src/object.c +++ b/src/object.c @@ -1073,10 +1073,10 @@ void objectCommand(client *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"help")) { const char *help[] = { -"encoding -- Return the kind of internal representation used in order to store the value associated with a key.", -"freq -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key.", -"idletime -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key.", -"refcount -- Return the number of references of the value associated with the specified key.", +"ENCODING -- Return the kind of internal representation used in order to store the value associated with a key.", +"FREQ -- Return the access frequency index of the key. The returned integer is proportional to the logarithm of the recent access frequency of the key.", +"IDLETIME -- Return the idle time of the key, that is the approximated number of seconds elapsed since the last access to the key.", +"REFCOUNT -- Return the number of references of the value associated with the specified key.", NULL }; addReplyHelp(c, help); From 1d9705a42431d55eed3b24f98f37f32b2a006939 Mon Sep 17 00:00:00 2001 From: Guy Benoish Date: Tue, 6 Mar 2018 19:34:44 +0700 Subject: [PATCH 007/616] Don't call sdscmp() with shared.maxstring or shared.minstring --- src/t_zset.c | 14 ++++++-------- tests/unit/type/zset.tcl | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index f7f4c6eb2..50652244b 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -519,12 +519,12 @@ int zslParseLexRangeItem(robj *item, sds *dest, int *ex) { switch(c[0]) { case '+': if (c[1] != '\0') return C_ERR; - *ex = 0; + *ex = 1; *dest = shared.maxstring; return C_OK; case '-': if (c[1] != '\0') return C_ERR; - *ex = 0; + *ex = 1; *dest = shared.minstring; return C_OK; case '(': @@ -597,9 +597,8 @@ int zslIsInLexRange(zskiplist *zsl, zlexrangespec *range) { zskiplistNode *x; /* Test for ranges that will always be empty. */ - if (sdscmplex(range->min,range->max) > 1 || - (sdscmp(range->min,range->max) == 0 && - (range->minex || range->maxex))) + int cmp = sdscmplex(range->min,range->max); + if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex))) return 0; x = zsl->tail; if (x == NULL || !zslLexValueGteMin(x->ele,range)) @@ -872,9 +871,8 @@ int zzlIsInLexRange(unsigned char *zl, zlexrangespec *range) { unsigned char *p; /* Test for ranges that will always be empty. */ - if (sdscmplex(range->min,range->max) > 1 || - (sdscmp(range->min,range->max) == 0 && - (range->minex || range->maxex))) + int cmp = sdscmplex(range->min,range->max); + if (cmp > 0 || (cmp == 0 && (range->minex || range->maxex))) return 0; p = ziplistIndex(zl,-2); /* Last element. */ diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 564825ae9..d8f3cfa53 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -388,7 +388,7 @@ start_server {tags {"zset"}} { 0 omega} } - test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZCOUNT basics" { + test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZLEXCOUNT basics" { create_default_lex_zset # inclusive range @@ -416,6 +416,22 @@ start_server {tags {"zset"}} { assert_equal {} [r zrevrangebylex zset \[elez \[elex] assert_equal {} [r zrevrangebylex zset (hill (omega] } + + test "ZLEXCOUNT advanced" { + create_default_lex_zset + + assert_equal 9 [r zlexcount zset - +] + assert_equal 0 [r zlexcount zset + -] + assert_equal 0 [r zlexcount zset + \[c] + assert_equal 0 [r zlexcount zset \[c -] + assert_equal 8 [r zlexcount zset \[bar +] + assert_equal 5 [r zlexcount zset \[bar \[foo] + assert_equal 4 [r zlexcount zset \[bar (foo] + assert_equal 4 [r zlexcount zset (bar \[foo] + assert_equal 3 [r zlexcount zset (bar (foo] + assert_equal 5 [r zlexcount zset - (foo] + assert_equal 1 [r zlexcount zset (maxstring +] + } test "ZRANGEBYSLEX with LIMIT" { create_default_lex_zset From d3e1f55c27699f94ac99aebcf9bf717183b4b8ff Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 29 Mar 2018 23:20:58 +0800 Subject: [PATCH 008/616] debug: avoid free client unexpectedly when reload & loadaof --- src/debug.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 293dbe36e..5d5029f5a 100644 --- a/src/debug.c +++ b/src/debug.c @@ -340,7 +340,10 @@ NULL return; } emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); - if (rdbLoad(server.rdb_filename,NULL) != C_OK) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + int ret = rdbLoad(server.rdb_filename,NULL); + aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + if (ret != C_OK) { addReplyError(c,"Error trying to load the RDB dump"); return; } @@ -349,7 +352,10 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { if (server.aof_state == AOF_ON) flushAppendOnlyFile(1); emptyDb(-1,EMPTYDB_NO_FLAGS,NULL); - if (loadAppendOnlyFile(server.aof_filename) != C_OK) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + int ret = loadAppendOnlyFile(server.aof_filename); + aeCreateFileEvent(server.el,c->fd,AE_READABLE,readQueryFromClient,c); + if (ret != C_OK) { addReply(c,shared.err); return; } From cf03a62ba78f1939b16a81d53c0de6d1aef73294 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 2 Apr 2018 18:36:17 +0300 Subject: [PATCH 009/616] Add redis-cli support for diskless replication (CAPA EOF) when setting repl-diskless-sync yes, and sending SYNC. redis-cli needs to be able to understand the EOF marker protocol in order to be able to skip or download the rdb file --- src/redis-cli.c | 112 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d80973e75..6fffa9024 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1793,9 +1793,31 @@ static void latencyDistMode(void) { * Slave mode *--------------------------------------------------------------------------- */ +#define RDB_EOF_MARK_SIZE 40 + +void sendReplconf(const char* arg1, const char* arg2) { + printf("sending REPLCONF %s %s\n", arg1, arg2); + redisReply *reply = redisCommand(context, "REPLCONF %s %s", arg1, arg2); + + /* Handle any error conditions */ + if(reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + exit(1); + } else if(reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "REPLCONF %s error: %s\n", arg1, reply->str); + /* non fatal, old versions may not support it */ + } + freeReplyObject(reply); +} + +void sendCapa() { + sendReplconf("capa", "eof"); +} + /* Sends SYNC and reads the number of bytes in the payload. Used both by - * slaveMode() and getRDB(). */ -unsigned long long sendSync(int fd) { + * slaveMode() and getRDB(). + * returns 0 in case an EOF marker is used. */ +unsigned long long sendSync(int fd, char *out_eof) { /* To start we need to send the SYNC command and return the payload. * The hiredis client lib does not understand this part of the protocol * and we don't want to mess with its buffers, so everything is performed @@ -1825,17 +1847,33 @@ unsigned long long sendSync(int fd) { printf("SYNC with master failed: %s\n", buf); exit(1); } + if (strncmp(buf+1,"EOF:",4) == 0 && strlen(buf+5) >= RDB_EOF_MARK_SIZE) { + memcpy(out_eof, buf+5, RDB_EOF_MARK_SIZE); + return 0; + } return strtoull(buf+1,NULL,10); } static void slaveMode(void) { int fd = context->fd; - unsigned long long payload = sendSync(fd); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(fd, eofmark); char buf[1024]; int original_output = config.output; - fprintf(stderr,"SYNC with master, discarding %llu " - "bytes of bulk transfer...\n", payload); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC with master, discarding " + "bytes of bulk transfer until EOF marker...\n"); + } else { + fprintf(stderr,"SYNC with master, discarding %llu " + "bytes of bulk transfer...\n", payload); + } + /* Discard the payload. */ while(payload) { @@ -1847,8 +1885,29 @@ static void slaveMode(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } } - fprintf(stderr,"SYNC done. Logging commands from master.\n"); + + if (usemark) { + unsigned long long offset = ULLONG_MAX - payload; + fprintf(stderr,"SYNC done after %llu bytes. Logging commands from master.\n", offset); + /* put the slave online */ + sleep(1); + sendReplconf("ACK", "0"); + } else + fprintf(stderr,"SYNC done. Logging commands from master.\n"); /* Now we can use hiredis to read the incoming protocol. */ config.output = OUTPUT_CSV; @@ -1865,11 +1924,22 @@ static void slaveMode(void) { static void getRDB(void) { int s = context->fd; int fd; - unsigned long long payload = sendSync(s); + static char eofmark[RDB_EOF_MARK_SIZE]; + static char lastbytes[RDB_EOF_MARK_SIZE]; + static int usemark = 0; + unsigned long long payload = sendSync(s, eofmark); char buf[4096]; - fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", - payload, config.rdb_filename); + if (payload == 0) { + payload = ULLONG_MAX; + memset(lastbytes,0,RDB_EOF_MARK_SIZE); + usemark = 1; + fprintf(stderr,"SYNC sent to master, writing bytes of bulk transfer until EOF marker to '%s'\n", + config.rdb_filename); + } else { + fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", + payload, config.rdb_filename); + } /* Write to file. */ if (!strcmp(config.rdb_filename,"-")) { @@ -1898,11 +1968,31 @@ static void getRDB(void) { exit(1); } payload -= nread; + + if (usemark) { + /* Update the last bytes array, and check if it matches our delimiter.*/ + if (nread >= RDB_EOF_MARK_SIZE) { + memcpy(lastbytes,buf+nread-RDB_EOF_MARK_SIZE,RDB_EOF_MARK_SIZE); + } else { + int rem = RDB_EOF_MARK_SIZE-nread; + memmove(lastbytes,lastbytes+nread,rem); + memcpy(lastbytes+rem,buf,nread); + } + if (memcmp(lastbytes,eofmark,RDB_EOF_MARK_SIZE) == 0) + break; + } + } + if (usemark) { + payload = ULLONG_MAX - payload - RDB_EOF_MARK_SIZE; + if (ftruncate(fd, payload) == -1) + fprintf(stderr,"ftruncate failed: %s.\n", strerror(errno)); + fprintf(stderr,"Transfer finished with success after %llu bytes\n", payload); + } else { + fprintf(stderr,"Transfer finished with success.\n"); } close(s); /* Close the file descriptor ASAP as fsync() may take time. */ fsync(fd); close(fd); - fprintf(stderr,"Transfer finished with success.\n"); exit(0); } @@ -2893,12 +2983,14 @@ int main(int argc, char **argv) { /* Slave mode */ if (config.slave_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); slaveMode(); } /* Get RDB mode. */ if (config.getrdb_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); + sendCapa(); getRDB(); } From 931e1d6d305484a4f38cc8781b5973ad0356c0c9 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Mon, 30 Apr 2018 19:33:01 +0300 Subject: [PATCH 010/616] Adds memory information about the script's cache to INFO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementation notes: as INFO is "already broken", I didn't want to break it further. Instead of computing the server.lua_script dict size on every call, I'm keeping a running sum of the body's length and dict overheads. This implementation is naive as it **does not** take into consideration dict rehashing, but that inaccuracy pays off in speed ;) Demo time: ```bash $ redis-cli info memory | grep "script" used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "" 0 ; redis-cli info memory | grep "script" (nil) used_memory_scripts:120 used_memory_scripts_human:120B number_of_cached_scripts:1 $ redis-cli script flush ; redis-cli info memory | grep "script" OK used_memory_scripts:96 used_memory_scripts_human:96B number_of_cached_scripts:0 $ redis-cli eval "return('Hello, Script Cache :)')" 0 ; redis-cli info memory | grep "script" "Hello, Script Cache :)" used_memory_scripts:152 used_memory_scripts_human:152B number_of_cached_scripts:1 $ redis-cli eval "return redis.sha1hex(\"return('Hello, Script Cache :)')\")" 0 ; redis-cli info memory | grep "script" "1be72729d43da5114929c1260a749073732dc822" used_memory_scripts:232 used_memory_scripts_human:232B number_of_cached_scripts:2 ✔ 19:03:54 redis [lua_scripts-in-info-memory L ✚…⚑] $ redis-cli evalsha 1be72729d43da5114929c1260a749073732dc822 0 "Hello, Script Cache :)" ``` --- src/scripting.c | 3 +++ src/server.c | 8 ++++++++ src/server.h | 1 + 3 files changed, 12 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 3c0597c7a..e131d8ae0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,6 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); + server.lua_scripts_mem = sizeof(dict); /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1073,6 +1074,7 @@ void scriptingInit(int setup) { * This function is used in order to reset the scripting environment. */ void scriptingRelease(void) { dictRelease(server.lua_scripts); + server.lua_scripts_mem = 0; lua_close(server.lua); } @@ -1207,6 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); + server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); incrRefCount(body); return sha; } diff --git a/src/server.c b/src/server.c index 7f51778d7..9bab389c0 100644 --- a/src/server.c +++ b/src/server.c @@ -2994,6 +2994,7 @@ sds genRedisInfoString(char *section) { char peak_hmem[64]; char total_system_hmem[64]; char used_memory_lua_hmem[64]; + char used_memory_scripts_hmem[64]; char used_memory_rss_hmem[64]; char maxmemory_hmem[64]; size_t zmalloc_used = zmalloc_used_memory(); @@ -3013,6 +3014,7 @@ sds genRedisInfoString(char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); + bytesToHuman(used_memory_scripts_hmem,server.lua_scripts_mem); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -3037,6 +3039,9 @@ sds genRedisInfoString(char *section) { "total_system_memory_human:%s\r\n" "used_memory_lua:%lld\r\n" "used_memory_lua_human:%s\r\n" + "used_memory_scripts:%lld\r\n" + "used_memory_scripts_human:%s\r\n" + "number_of_cached_scripts:%lu\r\n" "maxmemory:%lld\r\n" "maxmemory_human:%s\r\n" "maxmemory_policy:%s\r\n" @@ -3069,6 +3074,9 @@ sds genRedisInfoString(char *section) { total_system_hmem, memory_lua, used_memory_lua_hmem, + server.lua_scripts_mem, + used_memory_scripts_hmem, + dictSize(server.lua_scripts), server.maxmemory, maxmemory_hmem, evict_policy, diff --git a/src/server.h b/src/server.h index 0e9c3f285..260ba80d7 100644 --- a/src/server.h +++ b/src/server.h @@ -1203,6 +1203,7 @@ struct redisServer { client *lua_client; /* The "fake client" to query Redis from Lua */ client *lua_caller; /* The client running EVAL right now, or NULL */ dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ + unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ mstime_t lua_time_start; /* Start time of script, milliseconds time */ int lua_write_dirty; /* True if a write command was called during the From f34e96cdebf6e37309f6fa93572f87f6149e5587 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:49:00 +0200 Subject: [PATCH 011/616] clientsCronTrackExpansiveClients() skeleton and ideas. --- src/server.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/server.c b/src/server.c index 06826a356..53a8e5368 100644 --- a/src/server.c +++ b/src/server.c @@ -885,6 +885,28 @@ int clientsCronResizeQueryBuffer(client *c) { return 0; } +/* This function is used in order to track clients using the biggest amount + * of memory in the latest few seconds. This way we can provide such information + * in the INFO output (clients section), without having to do an O(N) scan for + * all the clients. + * + * This is how it works. We have an array of CLIENTS_PEAK_MEM_USAGE_SLOTS slots + * where we track, for each, the biggest client output and input buffers we + * saw in that slot. Every slot correspond to one of the latest seconds, since + * the array is indexed by doing UNIXTIME % CLIENTS_PEAK_MEM_USAGE_SLOTS. + * + * When we want to know what was recently the peak memory usage, we just scan + * such few slots searching for the maximum value. */ +#define CLIENTS_PEAK_MEM_USAGE_SLOTS 10 +size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; +size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; + +int clientsCronTrackExpansiveClients(client *c) { + size_t in_usage = sdsAllocSize(c->querybuf); + size_t out_usage = getClientOutputBufferMemoryUsage(c); + return 0; /* This function never terminates the client. */ +} + #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { /* Make sure to process at least numclients/server.hz of clients @@ -917,6 +939,7 @@ void clientsCron(void) { * terminated. */ if (clientsCronHandleTimeout(c,now)) continue; if (clientsCronResizeQueryBuffer(c)) continue; + if (clientsCronTrackExpansiveClients(c)) continue; } } From e88cf71692f9c8a11fbaa4cbe47af61bb9a1700c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:54:15 +0200 Subject: [PATCH 012/616] clientsCronTrackExpansiveClients() actual implementation. --- src/server.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 53a8e5368..b14d0740e 100644 --- a/src/server.c +++ b/src/server.c @@ -897,13 +897,26 @@ int clientsCronResizeQueryBuffer(client *c) { * * When we want to know what was recently the peak memory usage, we just scan * such few slots searching for the maximum value. */ -#define CLIENTS_PEAK_MEM_USAGE_SLOTS 10 +#define CLIENTS_PEAK_MEM_USAGE_SLOTS 8 size_t ClientsPeakMemInput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; size_t ClientsPeakMemOutput[CLIENTS_PEAK_MEM_USAGE_SLOTS]; int clientsCronTrackExpansiveClients(client *c) { size_t in_usage = sdsAllocSize(c->querybuf); size_t out_usage = getClientOutputBufferMemoryUsage(c); + int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS; + int j = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; + + /* Always zero the next sample, so that when we switch to that second, we'll + * only register samples that are greater in that second without considering + * the history of such slot. */ + ClientsPeakMemInput[j] = 0; + ClientsPeakMemOutput[j] = 0; + + /* Track the biggest values observed so far in this slot. */ + if (in_usage > ClientsPeakMemInput[i]) ClientsPeakMemInput[i] = in_usage; + if (out_usage > ClientsPeakMemOutput[i]) ClientsPeakMemOutput[i] = out_usage; + return 0; /* This function never terminates the client. */ } From d9eac8af47fc0cf953ee9ed1f5819164f19ec8fa Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:58:04 +0200 Subject: [PATCH 013/616] Implement a function to retrieve the expansive clients mem usage. --- src/server.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/server.c b/src/server.c index b14d0740e..15417a757 100644 --- a/src/server.c +++ b/src/server.c @@ -920,6 +920,18 @@ int clientsCronTrackExpansiveClients(client *c) { return 0; /* This function never terminates the client. */ } +/* Return the max samples in the memory usage of clients tracked by + * the function clientsCronTrackExpansiveClients(). */ +void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { + size_t i = 0, o = 0; + for (int j = 0; j < CLIENTS_PEAK_MEM_USAGE_SLOTS; j++) { + if (ClientsPeakMemInput[j] > i) i = ClientsPeakMemInput[j]; + if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[o]; + } + *in_usage = i; + *out_usage = o; +} + #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { /* Make sure to process at least numclients/server.hz of clients From 317245181008e4a90dce5944d8490ffcae6f1ab4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 13:59:13 +0200 Subject: [PATCH 014/616] Rename var in clientsCronTrackExpansiveClients() for clarity. --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 15417a757..3ece41b7e 100644 --- a/src/server.c +++ b/src/server.c @@ -905,13 +905,13 @@ int clientsCronTrackExpansiveClients(client *c) { size_t in_usage = sdsAllocSize(c->querybuf); size_t out_usage = getClientOutputBufferMemoryUsage(c); int i = server.unixtime % CLIENTS_PEAK_MEM_USAGE_SLOTS; - int j = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; + int zeroidx = (i+1) % CLIENTS_PEAK_MEM_USAGE_SLOTS; /* Always zero the next sample, so that when we switch to that second, we'll * only register samples that are greater in that second without considering * the history of such slot. */ - ClientsPeakMemInput[j] = 0; - ClientsPeakMemOutput[j] = 0; + ClientsPeakMemInput[zeroidx] = 0; + ClientsPeakMemOutput[zeroidx] = 0; /* Track the biggest values observed so far in this slot. */ if (in_usage > ClientsPeakMemInput[i]) ClientsPeakMemInput[i] = in_usage; From bad9fc8195df02b93f6dba5726ba425acbc58c75 Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Thu, 19 Jul 2018 20:48:08 +0800 Subject: [PATCH 015/616] Streams: ID of xclaim command starts from the sixth argument. --- src/t_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 201509c7e..abef73c43 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1988,7 +1988,7 @@ void xclaimCommand(client *c) { * the client successfully claimed some message, so it should be * executed in a "all or nothing" fashion. */ int j; - for (j = 4; j < c->argc; j++) { + for (j = 5; j < c->argc; j++) { streamID id; if (streamParseIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; } From 2ee63817934c852d2232d0b31c493769aa061931 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:16:19 +0200 Subject: [PATCH 016/616] Change INFO CLIENTS sections to report pre-computed max/min client buffers. --- src/server.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index 3ece41b7e..8b1cabecf 100644 --- a/src/server.c +++ b/src/server.c @@ -3081,17 +3081,17 @@ sds genRedisInfoString(char *section) { /* Clients */ if (allsections || defsections || !strcasecmp(section,"clients")) { - unsigned long lol, bib; - getClientsMaxBuffers(&lol,&bib); + size_t maxin, maxout; + getExpansiveClientsInfo(&maxin,&maxout); if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# Clients\r\n" "connected_clients:%lu\r\n" - "client_longest_output_list:%lu\r\n" - "client_biggest_input_buf:%lu\r\n" + "client_max_input_buffer:%zu\r\n" + "client_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n", listLength(server.clients)-listLength(server.slaves), - lol, bib, + maxin, maxout, server.blocked_clients); } From 3e2ee92a9f239e8aaeb40dc0d83b18001f73f3fb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:34:15 +0200 Subject: [PATCH 017/616] Fix wrong array index variable in getExpansiveClientsInfo(). --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 8b1cabecf..36f72f941 100644 --- a/src/server.c +++ b/src/server.c @@ -926,7 +926,7 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { size_t i = 0, o = 0; for (int j = 0; j < CLIENTS_PEAK_MEM_USAGE_SLOTS; j++) { if (ClientsPeakMemInput[j] > i) i = ClientsPeakMemInput[j]; - if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[o]; + if (ClientsPeakMemOutput[j] > o) o = ClientsPeakMemOutput[j]; } *in_usage = i; *out_usage = o; From cb09197bd3e3f607acb77befc9aedb2ac3420160 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Jul 2018 17:38:17 +0200 Subject: [PATCH 018/616] Rename INFO CLIENT max buffers field names for correctness. They are actually delayed a few seconds, so let's call them "recent". --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 36f72f941..8d49b5bf0 100644 --- a/src/server.c +++ b/src/server.c @@ -3087,8 +3087,8 @@ sds genRedisInfoString(char *section) { info = sdscatprintf(info, "# Clients\r\n" "connected_clients:%lu\r\n" - "client_max_input_buffer:%zu\r\n" - "client_max_output_buffer:%zu\r\n" + "client_recent_max_input_buffer:%zu\r\n" + "client_recent_max_output_buffer:%zu\r\n" "blocked_clients:%d\r\n", listLength(server.clients)-listLength(server.slaves), maxin, maxout, From e345c28a3e1b7a9b479a1cb11e3ac929db3f3add Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Jul 2018 09:36:44 +0200 Subject: [PATCH 019/616] Clarify that clientsCronTrackExpansiveClients() indexes may jump ahead. --- src/server.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 8d49b5bf0..1cb7a9e8e 100644 --- a/src/server.c +++ b/src/server.c @@ -909,7 +909,15 @@ int clientsCronTrackExpansiveClients(client *c) { /* Always zero the next sample, so that when we switch to that second, we'll * only register samples that are greater in that second without considering - * the history of such slot. */ + * the history of such slot. + * + * Note: our index may jump to any random position if serverCron() is not + * called for some reason with the normal frequency, for instance because + * some slow command is called taking multiple seconds to execute. In that + * case our array may end containing data which is potentially older + * than CLIENTS_PEAK_MEM_USAGE_SLOTS seconds: however this is not a problem + * since here we want just to track if "recently" there were very expansive + * clients from the POV of memory usage. */ ClientsPeakMemInput[zeroidx] = 0; ClientsPeakMemOutput[zeroidx] = 0; From b8e5d07583aedbece11240f5d54fb4fbd4ae283a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Jul 2018 09:46:18 +0200 Subject: [PATCH 020/616] Top comment clientsCron(). --- src/server.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/server.c b/src/server.c index 1cb7a9e8e..93c0a46fd 100644 --- a/src/server.c +++ b/src/server.c @@ -940,12 +940,27 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) { *out_usage = o; } +/* This function is called by serverCron() and is used in order to perform + * operations on clients that are important to perform constantly. For instance + * we use this function in order to disconnect clients after a timeout, including + * clients blocked in some blocking command with a non-zero timeout. + * + * The function makes some effort to process all the clients every second, even + * if this cannot be strictly guaranteed, since serverCron() may be called with + * an actual frequency lower than server.hz in case of latency events like slow + * commands. + * + * It is very important for this function, and the functions it calls, to be + * very fast: sometimes Redis has tens of hundreds of connected clients, and the + * default server.hz value is 10, so sometimes here we need to process thousands + * of clients per second, turning this function into a source of latency. + */ #define CLIENTS_CRON_MIN_ITERATIONS 5 void clientsCron(void) { - /* Make sure to process at least numclients/server.hz of clients - * per call. Since this function is called server.hz times per second - * we are sure that in the worst case we process all the clients in 1 - * second. */ + /* Try to process at least numclients/server.hz of clients + * per call. Since normally (if there are no big latency events) this + * function is called server.hz times per second, in the average case we + * process all the clients in 1 second. */ int numclients = listLength(server.clients); int iterations = numclients/server.hz; mstime_t now = mstime(); From 16e707372a9682a574f519192511639ce16ba4f9 Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Sat, 21 Jul 2018 08:48:51 +0800 Subject: [PATCH 021/616] Consider aof write error as well as rdb in lua script. --- src/scripting.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 328e3d681..23ff96c53 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -493,11 +493,21 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { { luaPushError(lua, shared.roslaveerr->ptr); goto cleanup; - } else if (server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) + } else if ((server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 && + server.lastbgsave_status == C_ERR) || + (server.aof_state != AOF_OFF && + server.aof_last_write_status == C_ERR)) { - luaPushError(lua, shared.bgsaveerr->ptr); + if (server.aof_last_write_status == C_OK) { + luaPushError(lua, shared.bgsaveerr->ptr); + } else { + sds aof_write_err = sdscatfmt(sdsempty(), + "-MISCONF Errors writing to the AOF file: %s\r\n", + strerror(server.aof_last_write_errno)); + luaPushError(lua, aof_write_err); + sdsfree(aof_write_err); + } goto cleanup; } } From eba8dbdce427572f9bd6a800ae0715e24c4a2fda Mon Sep 17 00:00:00 2001 From: WuYunlong Date: Sat, 21 Jul 2018 10:00:32 +0800 Subject: [PATCH 022/616] Do not migrate already expired keys. --- src/cluster.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index e568f68a6..5085c2a6a 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5146,6 +5146,8 @@ try_again: serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); } + int num_keys_non_expired = 0; + /* Create RESTORE payload and generate the protocol to call the command. */ for (j = 0; j < num_keys; j++) { long long ttl = 0; @@ -5153,8 +5155,10 @@ try_again: if (expireat != -1) { ttl = expireat-mstime(); + if (ttl < 0) continue; if (ttl < 1) ttl = 1; } + num_keys_non_expired++; serverAssertWithInfo(c,NULL, rioWriteBulkCount(&cmd,'*',replace ? 5 : 4)); @@ -5217,9 +5221,9 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ - if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); + if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys_non_expired+1)); - for (j = 0; j < num_keys; j++) { + for (j = 0; j < num_keys_non_expired; j++) { if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; From d6438a2cadd1717a4d0fa8d940ba19561d591560 Mon Sep 17 00:00:00 2001 From: dsomeshwar Date: Sat, 21 Jul 2018 23:42:08 +0530 Subject: [PATCH 023/616] removing redundant check --- src/util.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/util.c b/src/util.c index ca5435026..dd776162e 100644 --- a/src/util.c +++ b/src/util.c @@ -368,9 +368,6 @@ int string2ll(const char *s, size_t slen, long long *value) { if (p[0] >= '1' && p[0] <= '9') { v = p[0]-'0'; p++; plen++; - } else if (p[0] == '0' && slen == 1) { - *value = 0; - return 1; } else { return 0; } From b1b181dae346c8306763d0349533645975cfb17c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 22 Jul 2018 10:17:35 +0300 Subject: [PATCH 024/616] fix recursion typo in zmalloc_usable --- src/zmalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 0d9917510..308774d86 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -183,7 +183,7 @@ size_t zmalloc_size(void *ptr) { return size+PREFIX_SIZE; } size_t zmalloc_usable(void *ptr) { - return zmalloc_usable(ptr)-PREFIX_SIZE; + return zmalloc_size(ptr)-PREFIX_SIZE; } #endif From bcfc6bb7eeaf6893c4d328d5aa734af85efa1e88 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sun, 22 Jul 2018 17:52:09 +0800 Subject: [PATCH 025/616] Streams: skip master fileds only when we are going forward in streamIteratorGetID --- src/t_stream.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 201509c7e..15a41bedf 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -548,13 +548,16 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { si->master_fields_count = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ si->master_fields_start = si->lp_ele; - /* Skip master fileds to seek the first entry. */ - for (uint64_t i = 0; i < si->master_fields_count; i++) - si->lp_ele = lpNext(si->lp,si->lp_ele); - /* We are now pointing the zero term of the master entry. If + /* We are now pointing the start filed of the master entry. If * we are iterating in reverse order, we need to seek the * end of the listpack. */ - if (si->rev) si->lp_ele = lpLast(si->lp); + if (si->rev) { + si->lp_ele = lpLast(si->lp); + } else { + /* Skip master fileds to seek the first entry. */ + for (uint64_t i = 0; i < si->master_fields_count; i++) + si->lp_ele = lpNext(si->lp,si->lp_ele); + } } else if (si->rev) { /* If we are itereating in the reverse order, and this is not * the first entry emitted for this listpack, then we already From a3c4ec8a84f2203796eb0638e26e26d78556bf71 Mon Sep 17 00:00:00 2001 From: Itamar Haber Date: Sun, 22 Jul 2018 21:16:00 +0300 Subject: [PATCH 026/616] Adds Lua overheads to MEMORY STATS, smartens the MEMORY DOCTOR --- src/object.c | 29 +++++++++++++++++++++++++++-- src/scripting.c | 4 ++-- src/server.h | 1 + 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/object.c b/src/object.c index 82b82632b..f72b29dd9 100644 --- a/src/object.c +++ b/src/object.c @@ -986,6 +986,18 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; + mem = 0; + mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + + dictSlots(server.lua_scripts) * sizeof(dictEntry*); + mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + + dictSlots(server.repl_scriptcache_dict) * sizeof(dictEntry*); + if (listLength(server.repl_scriptcache_fifo) > 0) { + mem += listLength(server.repl_scriptcache_fifo) * (sizeof(listNode) + + sdsZmallocSize(listNodeValue(listFirst(server.repl_scriptcache_fifo)))); + } + mh->lua_caches = mem; + mem_total+=mem; + for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; long long keyscount = dictSize(db->dict); @@ -1043,6 +1055,7 @@ sds getMemoryDoctorReport(void) { int high_alloc_rss = 0; /* High rss overhead. */ int big_slave_buf = 0; /* Slave buffers are too big. */ int big_client_buf = 0; /* Client buffers are too big. */ + int many_scripts = 0; /* Script cache has too many scripts. */ int num_reports = 0; struct redisMemOverhead *mh = getMemoryOverheadData(); @@ -1093,6 +1106,12 @@ sds getMemoryDoctorReport(void) { big_slave_buf = 1; num_reports++; } + + /* Too many (over 42) scripts are cached? */ + if (dictSize(server.lua_scripts) > 42) { + many_scripts = 1; + num_reports++; + } } sds s; @@ -1122,7 +1141,7 @@ sds getMemoryDoctorReport(void) { s = sdscatprintf(s," * High allocator RSS overhead: This instance has an RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the allocator is much larger than the sum what the allocator actually holds). This problem is usually due to a large peak memory (check if there is a peak memory entry above in the report), you can try the MEMORY PURGE command to reclaim it.\n\n"); } if (high_proc_rss) { - s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to LUA scripts or Modules.\n\n"); + s = sdscatprintf(s," * High process RSS overhead: This instance has non-allocator RSS memory overhead is greater than 1.1 (this means that the Resident Set Size of the Redis process is much larger than the RSS the allocator holds). This problem may be due to Lua scripts or Modules.\n\n"); } if (big_slave_buf) { s = sdscat(s," * Big slave buffers: The slave output buffers in this instance are greater than 10MB for each slave (on average). This likely means that there is some slave instance that is struggling receiving data, either because it is too slow or because of networking issues. As a result, data piles on the master output buffers. Please try to identify what slave is not receiving data correctly and why. You can use the INFO output in order to check the slaves delays and the CLIENT LIST command to check the output buffers of each slave.\n\n"); @@ -1130,6 +1149,9 @@ sds getMemoryDoctorReport(void) { if (big_client_buf) { s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n"); } + if (many_scripts) { + s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 42). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); + } s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n"); } freeMemoryOverheadData(mh); @@ -1236,7 +1258,7 @@ void memoryCommand(client *c) { } else if (!strcasecmp(c->argv[1]->ptr,"stats") && c->argc == 2) { struct redisMemOverhead *mh = getMemoryOverheadData(); - addReplyMultiBulkLen(c,(24+mh->num_dbs)*2); + addReplyMultiBulkLen(c,(25+mh->num_dbs)*2); addReplyBulkCString(c,"peak.allocated"); addReplyLongLong(c,mh->peak_allocated); @@ -1259,6 +1281,9 @@ void memoryCommand(client *c) { addReplyBulkCString(c,"aof.buffer"); addReplyLongLong(c,mh->aof_buffer); + addReplyBulkCString(c,"lua.caches"); + addReplyLongLong(c,mh->lua_caches); + for (size_t j = 0; j < mh->num_dbs; j++) { char dbname[32]; snprintf(dbname,sizeof(dbname),"db.%zd",mh->db[j].dbid); diff --git a/src/scripting.c b/src/scripting.c index e131d8ae0..1a9cc1c60 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -919,7 +919,7 @@ void scriptingInit(int setup) { * This is useful for replication, as we need to replicate EVALSHA * as EVAL, so we need to remember the associated script. */ server.lua_scripts = dictCreate(&shaScriptObjectDictType,NULL); - server.lua_scripts_mem = sizeof(dict); + server.lua_scripts_mem = 0; /* Register the redis commands table and fields */ lua_newtable(lua); @@ -1209,7 +1209,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { * EVALSHA commands as EVAL using the original script. */ int retval = dictAdd(server.lua_scripts,sha,body); serverAssertWithInfo(c ? c : server.lua_client,NULL,retval == DICT_OK); - server.lua_scripts_mem += sdslen(body->ptr) + sizeof(dictEntry); + server.lua_scripts_mem += sdsZmallocSize(sha) + sdsZmallocSize(body->ptr); incrRefCount(body); return sha; } diff --git a/src/server.h b/src/server.h index 260ba80d7..3d9fa2307 100644 --- a/src/server.h +++ b/src/server.h @@ -834,6 +834,7 @@ struct redisMemOverhead { size_t clients_slaves; size_t clients_normal; size_t aof_buffer; + size_t lua_caches; size_t overhead_total; size_t dataset; size_t total_keys; From 2f58f65ee894949db56902c3292b827041dd12ec Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 14:13:58 +0200 Subject: [PATCH 027/616] Dynamic HZ: separate hz from the configured hz. This way we can remember what the user configured HZ is, but change the actual HZ dynamically if needed in the dynamic HZ feature implementation. --- src/config.c | 16 ++++++++-------- src/server.c | 5 ++++- src/server.h | 3 +++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/config.c b/src/config.c index 54494c8e1..5acea06d5 100644 --- a/src/config.c +++ b/src/config.c @@ -441,9 +441,9 @@ void loadServerConfigFromString(char *config) { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"hz") && argc == 2) { - server.hz = atoi(argv[1]); - if (server.hz < CONFIG_MIN_HZ) server.hz = CONFIG_MIN_HZ; - if (server.hz > CONFIG_MAX_HZ) server.hz = CONFIG_MAX_HZ; + server.config_hz = atoi(argv[1]); + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { int yes; @@ -1158,11 +1158,11 @@ void configSetCommand(client *c) { } config_set_numerical_field( "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { } config_set_numerical_field( - "hz",server.hz,0,INT_MAX) { + "hz",server.config_hz,0,INT_MAX) { /* Hz is more an hint from the user, so we accept values out of range * but cap them to reasonable values. */ - if (server.hz < CONFIG_MIN_HZ) server.hz = CONFIG_MIN_HZ; - if (server.hz > CONFIG_MAX_HZ) server.hz = CONFIG_MAX_HZ; + if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; + if (server.config_hz > CONFIG_MAX_HZ) server.config_hz = CONFIG_MAX_HZ; } config_set_numerical_field( "watchdog-period",ll,0,INT_MAX) { if (ll) @@ -1329,7 +1329,7 @@ void configGetCommand(client *c) { config_get_numerical_field("slave-announce-port",server.slave_announce_port); config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); - config_get_numerical_field("hz",server.hz); + config_get_numerical_field("hz",server.config_hz); config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); @@ -2098,7 +2098,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"activedefrag",server.active_defrag_enabled,CONFIG_DEFAULT_ACTIVE_DEFRAG); rewriteConfigYesNoOption(state,"protected-mode",server.protected_mode,CONFIG_DEFAULT_PROTECTED_MODE); rewriteConfigClientoutputbufferlimitOption(state); - rewriteConfigNumericalOption(state,"hz",server.hz,CONFIG_DEFAULT_HZ); + rewriteConfigNumericalOption(state,"hz",server.config_hz,CONFIG_DEFAULT_HZ); rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,CONFIG_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); rewriteConfigYesNoOption(state,"rdb-save-incremental-fsync",server.rdb_save_incremental_fsync,CONFIG_DEFAULT_RDB_SAVE_INCREMENTAL_FSYNC); rewriteConfigYesNoOption(state,"aof-load-truncated",server.aof_load_truncated,CONFIG_DEFAULT_AOF_LOAD_TRUNCATED); diff --git a/src/server.c b/src/server.c index 93c0a46fd..557af84d9 100644 --- a/src/server.c +++ b/src/server.c @@ -1512,7 +1512,7 @@ void initServerConfig(void) { server.timezone = timezone; /* Initialized by tzset(). */ server.configfile = NULL; server.executable = NULL; - server.hz = CONFIG_DEFAULT_HZ; + server.config_hz = CONFIG_DEFAULT_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = CONFIG_DEFAULT_SERVER_PORT; server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; @@ -1991,6 +1991,7 @@ void initServer(void) { server.syslog_facility); } + server.hz = server.config_hz; server.pid = getpid(); server.current_client = NULL; server.clients = listCreate(); @@ -3074,6 +3075,7 @@ sds genRedisInfoString(char *section) { "uptime_in_seconds:%jd\r\n" "uptime_in_days:%jd\r\n" "hz:%d\r\n" + "configured_hz:%d\r\n" "lru_clock:%ld\r\n" "executable:%s\r\n" "config_file:%s\r\n", @@ -3097,6 +3099,7 @@ sds genRedisInfoString(char *section) { (intmax_t)uptime, (intmax_t)(uptime/(3600*24)), server.hz, + server.config_hz, (unsigned long) lruclock, server.executable ? server.executable : "", server.configfile ? server.configfile : ""); diff --git a/src/server.h b/src/server.h index dae7561d8..4ac323da1 100644 --- a/src/server.h +++ b/src/server.h @@ -923,6 +923,9 @@ struct redisServer { char *configfile; /* Absolute config file path, or NULL */ char *executable; /* Absolute executable file path. */ char **exec_argv; /* Executable argv vector (copy). */ + int config_hz; /* Configured HZ value. May be different than + the actual 'hz' field value if dynamic-hz + is enabled. */ int hz; /* serverCron() calls frequency in hertz */ redisDb *db; dict *commands; /* Command table */ From 89c6c7bd88913d21788bbac3404ebbc70629c04c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 14:21:04 +0200 Subject: [PATCH 028/616] Dynamic HZ: adapt cron frequency to number of clients. --- src/server.c | 11 +++++++++++ src/server.h | 11 ++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index 557af84d9..aff7911e4 100644 --- a/src/server.c +++ b/src/server.c @@ -1096,6 +1096,17 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Update the time cache. */ updateCachedTime(); + /* Adapt the server.hz value to the number of configured clients. If we have + * many clients, we want to call serverCron() with an higher frequency. */ + server.hz = server.config_hz; + while (listLength(server.clients) / server.hz > MAX_CLIENTS_PER_CLOCK_TICK) { + server.hz *= 2; + if (server.hz > CONFIG_MAX_HZ) { + server.hz = CONFIG_MAX_HZ; + break; + } + } + run_with_period(100) { trackInstantaneousMetric(STATS_METRIC_COMMAND,server.stat_numcommands); trackInstantaneousMetric(STATS_METRIC_NET_INPUT, diff --git a/src/server.h b/src/server.h index 4ac323da1..a139b5440 100644 --- a/src/server.h +++ b/src/server.h @@ -78,12 +78,13 @@ typedef long long mstime_t; /* millisecond time type. */ #define C_ERR -1 /* Static server configuration */ -#define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ +#define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 #define CONFIG_MAX_HZ 500 -#define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port */ -#define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog */ -#define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* default client timeout: infinite */ +#define MAX_CLIENTS_PER_CLOCK_TICK 200 /* HZ is adapted based on that. */ +#define CONFIG_DEFAULT_SERVER_PORT 6379 /* TCP port. */ +#define CONFIG_DEFAULT_TCP_BACKLOG 511 /* TCP listen backlog. */ +#define CONFIG_DEFAULT_CLIENT_TIMEOUT 0 /* Default client timeout: infinite */ #define CONFIG_DEFAULT_DBNUM 16 #define CONFIG_MAX_LINE 1024 #define CRON_DBS_PER_CALL 16 @@ -91,7 +92,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define PROTO_SHARED_SELECT_CMDS 10 #define OBJ_SHARED_INTEGERS 10000 #define OBJ_SHARED_BULKHDR_LEN 32 -#define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages */ +#define LOG_MAX_LEN 1024 /* Default maximum length of syslog messages.*/ #define AOF_REWRITE_PERC 100 #define AOF_REWRITE_MIN_SIZE (64*1024*1024) #define AOF_REWRITE_ITEMS_PER_CMD 64 From d1e07d021ef5b0a00ffe117135bf89d4cc9cca10 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Jul 2018 18:44:38 +0200 Subject: [PATCH 029/616] Change 42 to 1000 as warning level for cached scripts. Related to #4883. --- src/object.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object.c b/src/object.c index c1f6ac4b2..406816140 100644 --- a/src/object.c +++ b/src/object.c @@ -1138,8 +1138,8 @@ sds getMemoryDoctorReport(void) { num_reports++; } - /* Too many (over 42) scripts are cached? */ - if (dictSize(server.lua_scripts) > 42) { + /* Too many scripts are cached? */ + if (dictSize(server.lua_scripts) > 1000) { many_scripts = 1; num_reports++; } @@ -1181,7 +1181,7 @@ sds getMemoryDoctorReport(void) { s = sdscat(s," * Big client buffers: The clients output buffers in this instance are greater than 200K per client (on average). This may result from different causes, like Pub/Sub clients subscribed to channels bot not receiving data fast enough, so that data piles on the Redis instance output buffer, or clients sending commands with large replies or very large sequences of commands in the same pipeline. Please use the CLIENT LIST command in order to investigate the issue if it causes problems in your instance, or to understand better why certain clients are using a big amount of memory.\n\n"); } if (many_scripts) { - s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 42). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); + s = sdscat(s," * Many scripts: There seem to be many cached scripts in this instance (more than 1000). This may be because scripts are generated and `EVAL`ed, instead of being parameterized (with KEYS and ARGV), `SCRIPT LOAD`ed and `EVALSHA`ed. Unless `SCRIPT FLUSH` is called periodically, the scripts' caches may end up consuming most of your memory.\n\n"); } s = sdscat(s,"I'm here to keep you safe, Sam. I want to help you.\n"); } From 83d3e979b67aa878374b28191bc113e3c6a96bd1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 00:18:39 +0200 Subject: [PATCH 030/616] string2ll(): test for NULL pointer in all the cases. --- src/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index ca5435026..02298e8df 100644 --- a/src/util.c +++ b/src/util.c @@ -369,7 +369,7 @@ int string2ll(const char *s, size_t slen, long long *value) { v = p[0]-'0'; p++; plen++; } else if (p[0] == '0' && slen == 1) { - *value = 0; + if (value != NULL) *value = 0; return 1; } else { return 0; From 82c80952d84a505f3beb8ae91139ad13e8a0833c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 00:20:20 +0200 Subject: [PATCH 031/616] string2ll(): remove duplicated check for special case. Related to #5157. The PR author correctly indentified that the check was duplicated, but removing the second one introduces a bug that was fixed in the past (hence the duplication). Instead we can remove the first instance of the check without issues. --- src/util.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/util.c b/src/util.c index 02298e8df..e2ada556a 100644 --- a/src/util.c +++ b/src/util.c @@ -349,12 +349,7 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen == slen) return 0; - /* Special case: first and only digit is 0. */ - if (slen == 1 && p[0] == '0') { - if (value != NULL) *value = 0; - return 1; - } - + /* Handle negative integers. */ if (p[0] == '-') { negative = 1; p++; plen++; From ac1e9e00e8e0fcde189d5c46046733a370cea2de Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 10:22:12 +0200 Subject: [PATCH 032/616] Restore string2ll() to original version. See PR #5157. --- src/util.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/util.c b/src/util.c index e2ada556a..ca5435026 100644 --- a/src/util.c +++ b/src/util.c @@ -349,7 +349,12 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen == slen) return 0; - /* Handle negative integers. */ + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return 1; + } + if (p[0] == '-') { negative = 1; p++; plen++; @@ -364,7 +369,7 @@ int string2ll(const char *s, size_t slen, long long *value) { v = p[0]-'0'; p++; plen++; } else if (p[0] == '0' && slen == 1) { - if (value != NULL) *value = 0; + *value = 0; return 1; } else { return 0; From e2065c7fabe15c301ab4dd8cd534659924104d7c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 1 Apr 2018 17:09:35 +0300 Subject: [PATCH 033/616] fix slave buffer test suite false positives it looks like on slow machines we're getting: [err]: slave buffer are counted correctly in tests/unit/maxmemory.tcl Expected condition '$slave_buf > 2*1024*1024' to be true (16914 > 2*1024*1024) this is a result of the slave waking up too early and eating the slave buffer before the traffic and the test ends. --- tests/unit/maxmemory.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index e3cd11114..7629fe05e 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -182,7 +182,7 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { # put the slave to sleep set rd_slave [redis_deferring_client] - $rd_slave debug sleep 60 + $rd_slave debug sleep 300 # send some 10mb woth of commands that don't increase the memory usage if {$pipeline == 1} { From b20ab15a40e2600e78156cba254857326ab00d6c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 10:27:20 +0200 Subject: [PATCH 034/616] string2ll(): better commenting. --- src/util.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/util.c b/src/util.c index dd776162e..3fa6c9244 100644 --- a/src/util.c +++ b/src/util.c @@ -346,6 +346,7 @@ int string2ll(const char *s, size_t slen, long long *value) { int negative = 0; unsigned long long v; + /* A zero length string is not a valid number. */ if (plen == slen) return 0; @@ -355,6 +356,8 @@ int string2ll(const char *s, size_t slen, long long *value) { return 1; } + /* Handle negative numbers: just set a flag and continue like if it + * was a positive number. Later convert into negative. */ if (p[0] == '-') { negative = 1; p++; plen++; @@ -372,6 +375,7 @@ int string2ll(const char *s, size_t slen, long long *value) { return 0; } + /* Parse all the other digits, checking for overflow at every step. */ while (plen < slen && p[0] >= '0' && p[0] <= '9') { if (v > (ULLONG_MAX / 10)) /* Overflow. */ return 0; @@ -388,6 +392,8 @@ int string2ll(const char *s, size_t slen, long long *value) { if (plen < slen) return 0; + /* Convert to negative if needed, and do the final overflow check when + * converting from unsigned long long to long long. */ if (negative) { if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ return 0; From d59ae2665655806d77bd3c1a4263fb0e17d6336b Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 11:07:27 +0200 Subject: [PATCH 035/616] Streams: refactoring of next entry seek in the iterator. After #5161 the code could be made a bit more obvious for newcomers. --- src/t_stream.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 15a41bedf..4566b9bf3 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -548,15 +548,19 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { si->master_fields_count = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ si->master_fields_start = si->lp_ele; - /* We are now pointing the start filed of the master entry. If - * we are iterating in reverse order, we need to seek the - * end of the listpack. */ - if (si->rev) { - si->lp_ele = lpLast(si->lp); - } else { - /* Skip master fileds to seek the first entry. */ + /* We are now pointing to the first field of the master entry. + * We need to seek either the first or the last entry depending + * on the direction of the iteration. */ + if (!si->rev) { + /* If we are iterating in normal order, skip the master fields + * to seek the first actual entry. */ for (uint64_t i = 0; i < si->master_fields_count; i++) si->lp_ele = lpNext(si->lp,si->lp_ele); + } else { + /* If we are iterating in reverse direction, just seek the + * last part of the last entry in the listpack (that is, the + * fields count). */ + si->lp_ele = lpLast(si->lp); } } else if (si->rev) { /* If we are itereating in the reverse order, and this is not From 61ca1840f1fdfb5bb78acc45971194b91065165e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 17:27:43 +0200 Subject: [PATCH 036/616] Make changes of PR #5154 hopefully simpler. --- src/cluster.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 5085c2a6a..19688d593 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5146,7 +5146,10 @@ try_again: serverAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); } - int num_keys_non_expired = 0; + int expired = 0; /* Number of keys that we'll find already expired. + Note that serializing large keys may take some time + so certain keys that were found non expired by the + lookupKey() function, may be expired later. */ /* Create RESTORE payload and generate the protocol to call the command. */ for (j = 0; j < num_keys; j++) { @@ -5155,10 +5158,12 @@ try_again: if (expireat != -1) { ttl = expireat-mstime(); - if (ttl < 0) continue; + if (ttl < 0) { + expired++; + continue; + } if (ttl < 1) ttl = 1; } - num_keys_non_expired++; serverAssertWithInfo(c,NULL, rioWriteBulkCount(&cmd,'*',replace ? 5 : 4)); @@ -5221,9 +5226,9 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ - if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys_non_expired+1)); + if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); - for (j = 0; j < num_keys_non_expired; j++) { + for (j = 0; j < num_keys-expired; j++) { if (syncReadLine(cs->fd, buf2, sizeof(buf2), timeout) <= 0) { socket_error = 1; break; From de401f46fd972abdaf1e786ed86fa5d236d1f3ae Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Jul 2018 17:31:39 +0200 Subject: [PATCH 037/616] Example the magic +1 in migrateCommand(). Related to #5154. --- src/cluster.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cluster.c b/src/cluster.c index 19688d593..8b76d9ad0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5226,6 +5226,10 @@ try_again: int socket_error = 0; int del_idx = 1; /* Index of the key argument for the replicated DEL op. */ + /* Allocate the new argument vector that will replace the current command, + * to propagate the MIGRATE as a DEL command (if no COPY option was given). + * We allocate num_keys+1 because the additional argument is for "DEL" + * command name itself. */ if (!copy) newargv = zmalloc(sizeof(robj*)*(num_keys+1)); for (j = 0; j < num_keys-expired; j++) { From 3ca7a87947fbf2d54f85bd51e85fe70188c6c04b Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 25 Jul 2018 00:06:27 +0800 Subject: [PATCH 038/616] Streams: fix xdel memory leak --- src/t_stream.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/t_stream.c b/src/t_stream.c index 7b1076b16..5a5263325 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -757,6 +757,7 @@ int streamDeleteItem(stream *s, streamID *id) { streamIteratorRemoveEntry(&si,&myid); deleted = 1; } + streamIteratorStop(&si); return deleted; } From 391a1cda58734f145487b6346f95700208293872 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 25 Jul 2018 18:13:34 +0800 Subject: [PATCH 039/616] optimize flushdb, avoid useless loops --- src/db.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index 055af71be..39686f1b6 100644 --- a/src/db.c +++ b/src/db.c @@ -329,7 +329,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { * database(s). Otherwise -1 is returned in the specific case the * DB number is out of range, and errno is set to EINVAL. */ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { - int j, async = (flags & EMPTYDB_ASYNC); + int async = (flags & EMPTYDB_ASYNC); long long removed = 0; if (dbnum < -1 || dbnum >= server.dbnum) { @@ -337,7 +337,10 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } - for (j = 0; j < server.dbnum; j++) { + int j = dbnum == -1 ? 0 : dbnum; + int dbmax = dbnum == -1 ? server.dbnum : dbnum+1; + + for (; j < dbmax; j++) { if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { From b2aaf15b7b8da08fd34060400ff357dc1911ff63 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Jul 2018 16:32:52 +0200 Subject: [PATCH 040/616] Make emptyDb() change introduced in #4852 simpler to read. --- src/db.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index 39686f1b6..a2890639b 100644 --- a/src/db.c +++ b/src/db.c @@ -337,10 +337,15 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { return -1; } - int j = dbnum == -1 ? 0 : dbnum; - int dbmax = dbnum == -1 ? server.dbnum : dbnum+1; + int startdb, enddb; + if (dbnum == -1) { + startdb = 0; + enddb = server.dbnum-1; + } else { + startdb = enddb = dbnum; + } - for (; j < dbmax; j++) { + for (int j = startdb; j <= enddb; j++) { if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { From 8acaa863c8291a34654729830bd470b4e2f83a1a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Jul 2018 16:34:46 +0200 Subject: [PATCH 041/616] Remove useless conditional from emptyDb(). Related to #4852. --- src/db.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/db.c b/src/db.c index a2890639b..6efe47011 100644 --- a/src/db.c +++ b/src/db.c @@ -346,7 +346,6 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) { } for (int j = startdb; j <= enddb; j++) { - if (dbnum != -1 && dbnum != j) continue; removed += dictSize(server.db[j].dict); if (async) { emptyDbAsync(&server.db[j]); From eade07dae7a8313fcc42c44f3c01c2597bcb6b84 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 29 Jul 2018 10:08:47 +0300 Subject: [PATCH 042/616] add DEBUG LOG, to to assist test suite debugging --- src/debug.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/debug.c b/src/debug.c index 32be3c59c..a66390dbb 100644 --- a/src/debug.c +++ b/src/debug.c @@ -290,6 +290,7 @@ void debugCommand(client *c) { "CRASH-AND-RECOVER -- Hard crash and restart after delay.", "DIGEST -- Output a hex signature representing the current DB content.", "ERROR -- Return a Redis protocol error with as message. Useful for clients unit tests to simulate Redis errors.", +"LOG -- write message to the server log.", "HTSTATS -- Return hash table statistics of the specified Redis database.", "HTSTATS-KEY -- Like htstats but for the hash table stored as key's value.", "LOADAOF -- Flush the AOF buffers on disk and reload the AOF in memory.", @@ -333,6 +334,9 @@ NULL } else if (!strcasecmp(c->argv[1]->ptr,"assert")) { if (c->argc >= 3) c->argv[2] = tryObjectEncoding(c->argv[2]); serverAssertWithInfo(c,c->argv[0],1 == 2); + } else if (!strcasecmp(c->argv[1]->ptr,"log") && c->argc == 3) { + serverLog(LL_WARNING, "DEBUG LOG: %s", (char*)c->argv[2]->ptr); + addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"reload")) { rdbSaveInfo rsi, *rsiptr; rsiptr = rdbPopulateSaveInfo(&rsi); From 1a722347b08c1dc21284120ae58dc57eacfeac1f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:37:30 +0200 Subject: [PATCH 043/616] Control dynamic HZ via server configuration. --- src/server.c | 17 +++++++++++------ src/server.h | 2 ++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/server.c b/src/server.c index 965cae967..23450c8fd 100644 --- a/src/server.c +++ b/src/server.c @@ -1096,14 +1096,18 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Update the time cache. */ updateCachedTime(); + server.hz = server.config_hz; /* Adapt the server.hz value to the number of configured clients. If we have * many clients, we want to call serverCron() with an higher frequency. */ - server.hz = server.config_hz; - while (listLength(server.clients) / server.hz > MAX_CLIENTS_PER_CLOCK_TICK) { - server.hz *= 2; - if (server.hz > CONFIG_MAX_HZ) { - server.hz = CONFIG_MAX_HZ; - break; + if (server.dynamic_hz) { + while (listLength(server.clients) / server.hz > + MAX_CLIENTS_PER_CLOCK_TICK) + { + server.hz *= 2; + if (server.hz > CONFIG_MAX_HZ) { + server.hz = CONFIG_MAX_HZ; + break; + } } } @@ -1524,6 +1528,7 @@ void initServerConfig(void) { server.configfile = NULL; server.executable = NULL; server.config_hz = CONFIG_DEFAULT_HZ; + server.dynamic_hz = CONFIG_DEFAULT_DYNAMIC_HZ; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = CONFIG_DEFAULT_SERVER_PORT; server.tcp_backlog = CONFIG_DEFAULT_TCP_BACKLOG; diff --git a/src/server.h b/src/server.h index 4d63faa66..21e4c4279 100644 --- a/src/server.h +++ b/src/server.h @@ -78,6 +78,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define C_ERR -1 /* Static server configuration */ +#define CONFIG_DEFAULT_DYNAMIC_HZ 1 /* Adapt hz to # of clients.*/ #define CONFIG_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ #define CONFIG_MIN_HZ 1 #define CONFIG_MAX_HZ 500 @@ -925,6 +926,7 @@ struct redisServer { char *configfile; /* Absolute config file path, or NULL */ char *executable; /* Absolute executable file path. */ char **exec_argv; /* Executable argv vector (copy). */ + int dynamic_hz; /* Change hz value depending on # of clients. */ int config_hz; /* Configured HZ value. May be different than the actual 'hz' field value if dynamic-hz is enabled. */ From 790afbcbe446e80980b0960e118b3531f2f86706 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:44:52 +0200 Subject: [PATCH 044/616] Make dynamic hz actually configurable. --- src/config.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config.c b/src/config.c index 5acea06d5..7bd9592b2 100644 --- a/src/config.c +++ b/src/config.c @@ -440,6 +440,10 @@ void loadServerConfigFromString(char *config) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"dynamic-hz") && argc == 2) { + if ((server.dynamic_hz = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"hz") && argc == 2) { server.config_hz = atoi(argv[1]); if (server.config_hz < CONFIG_MIN_HZ) server.config_hz = CONFIG_MIN_HZ; @@ -1072,6 +1076,8 @@ void configSetCommand(client *c) { "slave-lazy-flush",server.repl_slave_lazy_flush) { } config_set_bool_field( "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { + } config_set_bool_field( + "dynamic-hz",server.dynamic_hz) { /* Numerical fields. * config_set_numerical_field(name,var,min,max) */ @@ -1375,6 +1381,8 @@ void configGetCommand(client *c) { server.lazyfree_lazy_server_del); config_get_bool_field("slave-lazy-flush", server.repl_slave_lazy_flush); + config_get_bool_field("dynamic-hz", + server.dynamic_hz); /* Enum values */ config_get_enum_field("maxmemory-policy", @@ -2108,6 +2116,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); rewriteConfigYesNoOption(state,"slave-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); + rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); /* Rewrite Sentinel config if in Sentinel mode. */ if (server.sentinel_mode) rewriteConfigSentinelOption(state); From 2d699ec557e9b9debf4b9237f2924ead993870fd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 13:50:15 +0200 Subject: [PATCH 045/616] Document dynamic-hz in the example redis.conf. --- redis.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/redis.conf b/redis.conf index 42d24f26e..5cbc74bbd 100644 --- a/redis.conf +++ b/redis.conf @@ -1205,6 +1205,22 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# Normally it is useful to have an HZ value which is proportional to the +# number of clients connected. This is useful in order, for instance, to +# avoid too many clients are processed for each background task invocation +# in order to avoid latency spikes. +# +# Since the default HZ value by default is conservatively set to 10, Redis +# offers, and enables by default, the ability to use an adaptive HZ value +# which will temporary raise when there are many connected clients. +# +# When dynamic HZ is enabled, the actual configured HZ will be used as +# as a baseline, but multiples of the configured HZ value will be actually +# used as needed once more clients are connected. In this way an idle +# instance will use very little CPU time while a busy instance will be +# more responsive. +dynamic-hz yes + # When a child rewrites the AOF file, if the following option is enabled # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid From 5946410d7b97a299ff62e1527b9ad99e4d53090e Mon Sep 17 00:00:00 2001 From: Guy Korland Date: Mon, 30 Jul 2018 16:18:56 +0300 Subject: [PATCH 046/616] Few typo fixes --- src/module.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/module.c b/src/module.c index 9809cd74e..3735cbe24 100644 --- a/src/module.c +++ b/src/module.c @@ -548,7 +548,7 @@ void RM_KeyAtPos(RedisModuleCtx *ctx, int pos) { ctx->keys_pos[ctx->keys_count++] = pos; } -/* Helper for RM_CreateCommand(). Truns a string representing command +/* Helper for RM_CreateCommand(). Turns a string representing command * flags into the command flags used by the Redis core. * * It returns the set of flags, or -1 if unknown flags are found. */ @@ -595,7 +595,7 @@ int commandFlagsFromString(char *s) { * And is supposed to always return REDISMODULE_OK. * * The set of flags 'strflags' specify the behavior of the command, and should - * be passed as a C string compoesd of space separated words, like for + * be passed as a C string composed of space separated words, like for * example "write deny-oom". The set of flags are: * * * **"write"**: The command may modify the data set (it may also read @@ -616,7 +616,7 @@ int commandFlagsFromString(char *s) { * * **"allow-stale"**: The command is allowed to run on slaves that don't * serve stale data. Don't use if you don't know what * this means. - * * **"no-monitor"**: Don't propoagate the command on monitor. Use this if + * * **"no-monitor"**: Don't propagate the command on monitor. Use this if * the command has sensible data among the arguments. * * **"fast"**: The command time complexity is not greater * than O(log(N)) where N is the size of the collection or @@ -956,9 +956,9 @@ RedisModuleString *moduleAssertUnsharedString(RedisModuleString *str) { return str; } -/* Append the specified buffere to the string 'str'. The string must be a +/* Append the specified buffer to the string 'str'. The string must be a * string created by the user that is referenced only a single time, otherwise - * REDISMODULE_ERR is returend and the operation is not performed. */ + * REDISMODULE_ERR is returned and the operation is not performed. */ int RM_StringAppendBuffer(RedisModuleCtx *ctx, RedisModuleString *str, const char *buf, size_t len) { UNUSED(ctx); str = moduleAssertUnsharedString(str); @@ -1118,7 +1118,7 @@ int RM_ReplyWithArray(RedisModuleCtx *ctx, long len) { * * Note that in the above example there is no reason to postpone the array * length, since we produce a fixed number of elements, but in the practice - * the code may use an interator or other ways of creating the output so + * the code may use an iterator or other ways of creating the output so * that is not easy to calculate in advance the number of elements. */ void RM_ReplySetArrayLength(RedisModuleCtx *ctx, long len) { @@ -1410,7 +1410,7 @@ int RM_SelectDb(RedisModuleCtx *ctx, int newid) { * to call other APIs with the key handle as argument to perform * operations on the key. * - * The return value is the handle repesenting the key, that must be + * The return value is the handle representing the key, that must be * closed with RM_CloseKey(). * * If the key does not exist and WRITE mode is requested, the handle @@ -1664,7 +1664,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { * Key API for List type * -------------------------------------------------------------------------- */ -/* Push an element into a list, on head or tail depending on 'where' argumnet. +/* Push an element into a list, on head or tail depending on 'where' argument. * If the key pointer is about an empty key opened for writing, the key * is created. On error (key opened for read-only operations or of the wrong * type) REDISMODULE_ERR is returned, otherwise REDISMODULE_OK is returned. */ @@ -1769,7 +1769,7 @@ int RM_ZsetAdd(RedisModuleKey *key, double score, RedisModuleString *ele, int *f * The input and output flags, and the return value, have the same exact * meaning, with the only difference that this function will return * REDISMODULE_ERR even when 'score' is a valid double number, but adding it - * to the existing score resuts into a NaN (not a number) condition. + * to the existing score results into a NaN (not a number) condition. * * This function has an additional field 'newscore', if not NULL is filled * with the new score of the element after the increment, if no error @@ -3793,11 +3793,11 @@ void moduleReleaseGIL(void) { * -------------------------------------------------------------------------- */ /* Subscribe to keyspace notifications. This is a low-level version of the - * keyspace-notifications API. A module cand register callbacks to be notified + * keyspace-notifications API. A module can register callbacks to be notified * when keyspce events occur. * * Notification events are filtered by their type (string events, set events, - * etc), and the subsriber callback receives only events that match a specific + * etc), and the subscriber callback receives only events that match a specific * mask of event types. * * When subscribing to notifications with RedisModule_SubscribeToKeyspaceEvents @@ -3832,7 +3832,7 @@ void moduleReleaseGIL(void) { * used to send anything to the client, and has the db number where the event * occurred as its selected db number. * - * Notice that it is not necessary to enable norifications in redis.conf for + * Notice that it is not necessary to enable notifications in redis.conf for * module notifications to work. * * Warning: the notification callbacks are performed in a synchronous manner, @@ -3876,7 +3876,7 @@ void moduleNotifyKeyspaceEvent(int type, const char *event, robj *key, int dbid) ctx.client = moduleKeyspaceSubscribersClient; selectDb(ctx.client, dbid); - /* mark the handler as activer to avoid reentrant loops. + /* mark the handler as active to avoid reentrant loops. * If the subscriber performs an action triggering itself, * it will not be notified about it. */ sub->active = 1; From 06226f494dc552ecf6ffafc4ce4e5136b00c1009 Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Mon, 30 Jul 2018 21:32:07 +0800 Subject: [PATCH 047/616] Streams: add mmid_supp argument in streamParseIDOrReply(). If 'mmid_supp' is set to 0, "-" and "+" will be treated as an invalid ID. --- src/t_stream.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index ea1290faf..8725542c9 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1060,14 +1060,18 @@ int string2ull(const char *s, unsigned long long *value) { * form, just stating the milliseconds time part of the stream. In such a case * the missing part is set according to the value of 'missing_seq' parameter. * The IDs "-" and "+" specify respectively the minimum and maximum IDs - * that can be represented. + * that can be represented. If 'mmid_supp' is set to 0, "-" and "+" will be + * treated as an invalid ID. * * If 'c' is set to NULL, no reply is sent to the client. */ -int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { +int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int mmid_supp) { char buf[128]; if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid; memcpy(buf,o->ptr,sdslen(o->ptr)+1); + if (!mmid_supp && (buf[0] == '-' || buf[0] == '+') && + buf[1] == '\0') goto invalid; + /* Handle the "-" and "+" special cases. */ if (buf[0] == '-' && buf[1] == '\0') { id->ms = 0; From 9a4ef7eae7b9ed0bbda45b8216c3589daa91d65e Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Mon, 30 Jul 2018 21:33:01 +0800 Subject: [PATCH 048/616] Streams: rearrange the usage of '-' and '+' IDs in stream commands. --- src/t_stream.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 8725542c9..f233785ae 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1137,7 +1137,7 @@ void xaddCommand(client *c) { maxlen_arg_idx = i; } else { /* If we are here is a syntax error or a valid ID. */ - if (streamParseIDOrReply(c,c->argv[i],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[i],&id,0,0) != C_OK) return; id_given = 1; break; } @@ -1206,8 +1206,8 @@ void xrangeGenericCommand(client *c, int rev) { robj *startarg = rev ? c->argv[3] : c->argv[2]; robj *endarg = rev ? c->argv[2] : c->argv[3]; - if (streamParseIDOrReply(c,startarg,&startid,0) == C_ERR) return; - if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX) == C_ERR) return; + if (streamParseIDOrReply(c,startarg,&startid,0,1) == C_ERR) return; + if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX,1) == C_ERR) return; /* Parse the COUNT option if any. */ if (c->argc > 4) { @@ -1393,7 +1393,7 @@ void xreadCommand(client *c) { ids[id_idx].seq = UINT64_MAX; continue; } - if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK) + if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0,0) != C_OK) goto cleanup; } @@ -1667,7 +1667,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0,0) != C_OK) { return; } streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); @@ -1684,7 +1684,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0,1) != C_OK) { return; } cg->last_id = id; @@ -1744,7 +1744,7 @@ void xackCommand(client *c) { for (int j = 3; j < c->argc; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL: it will have a reference to the @@ -1791,9 +1791,9 @@ void xpendingCommand(client *c) { if (c->argc >= 6) { if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR) + if (streamParseIDOrReply(c,c->argv[3],&startid,0,1) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR) + if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX,1) == C_ERR) return; } @@ -2002,7 +2002,7 @@ void xclaimCommand(client *c) { int j; for (j = 4; j < c->argc; j++) { streamID id; - if (streamParseIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; + if (streamParseIDOrReply(NULL,c->argv[j],&id,0,0) != C_OK) break; } int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */ @@ -2061,7 +2061,7 @@ void xclaimCommand(client *c) { for (int j = 5; j <= last_id_arg; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL. */ @@ -2144,13 +2144,13 @@ void xdelCommand(client *c) { * executed because at some point an invalid ID is parsed. */ streamID id; for (int j = 2; j < c->argc; j++) { - if (streamParseIDOrReply(c,c->argv[j],&id,0) != C_OK) return; + if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; } /* Actually apply the command. */ int deleted = 0; for (int j = 2; j < c->argc; j++) { - streamParseIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */ + streamParseIDOrReply(c,c->argv[j],&id,0,0); /* Retval already checked. */ deleted += streamDeleteItem(s,&id); } From c49312cfdab3ba8e7e5ab72e888956155114f5bc Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Jul 2018 17:42:30 +0200 Subject: [PATCH 049/616] Add year in log. User: "is there a reason why redis server logs are missing the year in the "date time"?" Me: "I guess I did not imagine it would be stable enough to run for several years". --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 23450c8fd..3b2e8d2df 100644 --- a/src/server.c +++ b/src/server.c @@ -357,7 +357,7 @@ void serverLogRaw(int level, const char *msg) { gettimeofday(&tv,NULL); struct tm tm; nolocks_localtime(&tm,tv.tv_sec,server.timezone,server.daylight_active); - off = strftime(buf,sizeof(buf),"%d %b %H:%M:%S.",&tm); + off = strftime(buf,sizeof(buf),"%d %b %Y %H:%M:%S.",&tm); snprintf(buf+off,sizeof(buf)-off,"%03d",(int)tv.tv_usec/1000); if (server.sentinel_mode) { role_char = 'X'; /* Sentinel. */ From a6f173525f2d3080aa7a40b86f4808d35b203003 Mon Sep 17 00:00:00 2001 From: Pavel Rochnyack Date: Wed, 25 Jul 2018 21:50:24 +0700 Subject: [PATCH 050/616] INFO CPU: higher precision of reported values Closes: #5033 --- src/server.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server.c b/src/server.c index 3b2e8d2df..c0ea66282 100644 --- a/src/server.c +++ b/src/server.c @@ -3536,14 +3536,14 @@ sds genRedisInfoString(char *section) { if (sections++) info = sdscat(info,"\r\n"); info = sdscatprintf(info, "# CPU\r\n" - "used_cpu_sys:%.2f\r\n" - "used_cpu_user:%.2f\r\n" - "used_cpu_sys_children:%.2f\r\n" - "used_cpu_user_children:%.2f\r\n", - (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000, - (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000, - (float)c_ru.ru_stime.tv_sec+(float)c_ru.ru_stime.tv_usec/1000000, - (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000); + "used_cpu_sys:%ld.%06ld\r\n" + "used_cpu_user:%ld.%06ld\r\n" + "used_cpu_sys_children:%ld.%06ld\r\n" + "used_cpu_user_children:%ld.%06ld\r\n", + (long)self_ru.ru_stime.tv_sec, (long)self_ru.ru_stime.tv_usec, + (long)self_ru.ru_utime.tv_sec, (long)self_ru.ru_utime.tv_usec, + (long)c_ru.ru_stime.tv_sec, (long)c_ru.ru_stime.tv_usec, + (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec); } /* Command statistics */ From 0b04bf8419a63063a652732037fb7c8ebf79ee6e Mon Sep 17 00:00:00 2001 From: Salvatore Sanfilippo Date: Mon, 30 Jul 2018 18:03:15 +0200 Subject: [PATCH 051/616] Merge pull request #5168 from rpv-tomsk/issue-5033 INFO CPU: higher precision of reported values From dfdeb6a036f52c0b79ed85961c5eb05576630f84 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 30 Jul 2018 16:43:21 +0300 Subject: [PATCH 052/616] test suite conveniency improvements * allowing --single to be repeated * adding --only so that only a specific test inside a unit can be run * adding --skiptill useful to resume a test that crashed passed the problematic unit. useful together with --clients 1 * adding --skipfile to use a file containing list of tests names to skip * printing the names of the tests that are skiped by skipfile or denytags * adding --config to add config file options from command line --- runtest | 2 +- tests/support/test.tcl | 20 ++++++++++++++ tests/test_helper.tcl | 60 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/runtest b/runtest index d8451df57..ade1bd09a 100755 --- a/runtest +++ b/runtest @@ -11,4 +11,4 @@ then echo "You need tcl 8.5 or newer in order to run the Redis test" exit 1 fi -$TCLSH tests/test_helper.tcl $* +$TCLSH tests/test_helper.tcl "${@}" diff --git a/tests/support/test.tcl b/tests/support/test.tcl index d60eb3c47..6f02f2f12 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -1,6 +1,8 @@ set ::num_tests 0 set ::num_passed 0 set ::num_failed 0 +set ::num_skipped 0 +set ::num_aborted 0 set ::tests_failed {} proc fail {msg} { @@ -68,10 +70,26 @@ proc test {name code {okpattern undefined}} { # abort if tagged with a tag to deny foreach tag $::denytags { if {[lsearch $::tags $tag] >= 0} { + incr ::num_aborted + send_data_packet $::test_server_fd ignore $name return } } + # abort if test name in skiptests + if {[lsearch $::skiptests $name] >= 0} { + incr ::num_skipped + send_data_packet $::test_server_fd skip $name + return + } + + # abort if test name in skiptests + if {[llength $::only_tests] > 0 && [lsearch $::only_tests $name] < 0} { + incr ::num_skipped + send_data_packet $::test_server_fd skip $name + return + } + # check if tagged with at least 1 tag to allow when there *is* a list # of tags to allow, because default policy is to run everything if {[llength $::allowtags] > 0} { @@ -82,6 +100,8 @@ proc test {name code {okpattern undefined}} { } } if {$matched < 1} { + incr ::num_aborted + send_data_packet $::test_server_fd ignore $name return } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 720a428a6..ba3dce71c 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -74,7 +74,11 @@ set ::stack_logging 0 set ::verbose 0 set ::quiet 0 set ::denytags {} +set ::skiptests {} set ::allowtags {} +set ::only_tests {} +set ::single_tests {} +set ::skip_till "" set ::external 0; # If "1" this means, we are running against external instance set ::file ""; # If set, runs only the tests in this comma separated list set ::curfile ""; # Hold the filename of the current suite @@ -255,6 +259,8 @@ proc accept_test_clients {fd addr port} { # testing: just used to signal that a given test started. # ok: a test was executed with success. # err: a test was executed with an error. +# skip: a test was skipped by skipfile or individual test options. +# ignore: a test was skipped by a group tag. # exception: there was a runtime exception while executing the test. # done: all the specified test file was processed, this test client is # ready to accept a new task. @@ -283,6 +289,14 @@ proc read_from_test_client fd { puts "\[[colorstr green $status]\]: $data" } set ::active_clients_task($fd) "(OK) $data" + } elseif {$status eq {skip}} { + if {!$::quiet} { + puts "\[[colorstr yellow $status]\]: $data" + } + } elseif {$status eq {ignore}} { + if {!$::quiet} { + puts "\[[colorstr cyan $status]\]: $data" + } } elseif {$status eq {err}} { set err "\[[colorstr red $status]\]: $data" puts $err @@ -412,11 +426,15 @@ proc print_help_screen {} { "--stack-logging Enable OSX leaks/malloc stack logging." "--accurate Run slow randomized tests for more iterations." "--quiet Don't show individual tests." - "--single Just execute the specified unit (see next option)." + "--single Just execute the specified unit (see next option). this option can be repeated." "--list-tests List all the available test units." + "--only Just execute the specified test by test name. this option can be repeated." + "--skiptill Skip all units until (and including) the specified one." "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." + "--config extra config file argument" + "--skipfile name of a file containing test names that should be skipped (one per line)" "--dont-clean don't delete redis log files after the run" "--wait-server wait after server is started (so that you can attach a debugger)" "--help Print this help screen." @@ -436,6 +454,18 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } incr j + } elseif {$opt eq {--config}} { + set arg2 [lindex $argv [expr $j+2]] + lappend ::global_overrides $arg + lappend ::global_overrides $arg2 + incr j + incr j + } elseif {$opt eq {--skipfile}} { + incr j + set fp [open $arg r] + set file_data [read $fp] + close $fp + set ::skiptests [split $file_data "\n"] } elseif {$opt eq {--valgrind}} { set ::valgrind 1 } elseif {$opt eq {--stack-logging}} { @@ -456,7 +486,13 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } elseif {$opt eq {--force-failure}} { set ::force_failure 1 } elseif {$opt eq {--single}} { - set ::all_tests $arg + lappend ::single_tests $arg + incr j + } elseif {$opt eq {--only}} { + lappend ::only_tests $arg + incr j + } elseif {$opt eq {--skiptill}} { + set ::skip_till $arg incr j } elseif {$opt eq {--list-tests}} { foreach t $::all_tests { @@ -488,6 +524,26 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +if {$::skip_till != ""} { + set skipping 1 + foreach t $::all_tests { + if {$skipping == 0} { + lappend ::single_tests $t + } + if {$t == $::skip_till} { + set skipping 0 + } + } + if {$skipping} { + puts "test $::skip_till not found" + exit 0 + } +} + +if {[llength $::single_tests] > 0} { + set ::all_tests $::single_tests +} + proc attach_to_replication_stream {} { set s [socket [srv 0 "host"] [srv 0 "port"]] fconfigure $s -translation binary From b880a352d6b4839bc202e6e1fbf6c5cc89e25091 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 31 Jul 2018 12:07:57 +0800 Subject: [PATCH 053/616] refactor dbOverwrite to make lazyfree work --- src/db.c | 17 +++++++++++------ src/lazyfree.c | 11 +++++++++++ src/server.h | 1 + src/t_set.c | 10 ++++------ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/db.c b/src/db.c index 6efe47011..ec92a2b4e 100644 --- a/src/db.c +++ b/src/db.c @@ -184,14 +184,19 @@ void dbOverwrite(redisDb *db, robj *key, robj *val) { dictEntry *de = dictFind(db->dict,key->ptr); serverAssertWithInfo(NULL,key,de != NULL); + dictEntry auxentry = *de; + robj *old = dictGetVal(de); if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { - robj *old = dictGetVal(de); - int saved_lru = old->lru; - dictReplace(db->dict, key->ptr, val); - val->lru = saved_lru; - } else { - dictReplace(db->dict, key->ptr, val); + val->lru = old->lru; } + dictSetVal(db->dict, de, val); + + if (server.lazyfree_lazy_server_del) { + freeObjAsync(old); + dictSetVal(db->dict, &auxentry, NULL); + } + + dictFreeVal(db->dict, &auxentry); } /* High level Set operation. This function can be used in order to set diff --git a/src/lazyfree.c b/src/lazyfree.c index ac8a6bee9..3d3159c90 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -90,6 +90,17 @@ int dbAsyncDelete(redisDb *db, robj *key) { } } +/* Free an object, if the object is huge enough, free it in async way. */ +void freeObjAsync(robj *o) { + size_t free_effort = lazyfreeGetFreeEffort(o); + if (free_effort > LAZYFREE_THRESHOLD && o->refcount == 1) { + atomicIncr(lazyfree_objects,1); + bioCreateBackgroundJob(BIO_LAZY_FREE,o,NULL,NULL); + } else { + decrRefCount(o); + } +} + /* Empty a Redis DB asynchronously. What the function does actually is to * create a new empty set of hash tables and scheduling the old ones for * lazy freeing. */ diff --git a/src/server.h b/src/server.h index 14aed7fc0..a7985a39d 100644 --- a/src/server.h +++ b/src/server.h @@ -1824,6 +1824,7 @@ int dbAsyncDelete(redisDb *db, robj *key); void emptyDbAsync(redisDb *db); void slotToKeyFlushAsync(void); size_t lazyfreeGetPendingObjectsCount(void); +void freeObjAsync(robj *o); /* API to get key arguments from commands */ int *getKeysFromCommand(struct redisCommand *cmd, robj **argv, int argc, int *numkeys); diff --git a/src/t_set.c b/src/t_set.c index 8f21f71b1..5d48e3c43 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -516,11 +516,7 @@ void spopWithCountCommand(client *c) { sdsfree(sdsele); } - /* Assign the new set as the key value. */ - incrRefCount(set); /* Protect the old set value. */ - dbOverwrite(c->db,c->argv[1],newset); - - /* Tranfer the old set to the client and release it. */ + /* Tranfer the old set to the client. */ setTypeIterator *si; si = setTypeInitIterator(set); while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) { @@ -539,7 +535,9 @@ void spopWithCountCommand(client *c) { decrRefCount(objele); } setTypeReleaseIterator(si); - decrRefCount(set); + + /* Assign the new set as the key value. */ + dbOverwrite(c->db,c->argv[1],newset); } /* Don't propagate the command itself even if we incremented the From 0497fb5a0047bd938ba144d55b9edf40b8b9db90 Mon Sep 17 00:00:00 2001 From: shenlongxing Date: Tue, 31 Jul 2018 16:01:44 +0800 Subject: [PATCH 054/616] Fix cluster-announce-ip memory leak --- src/cluster.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 8b76d9ad0..7a6a5f782 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3325,10 +3325,11 @@ void clusterCron(void) { if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; if (changed) { - prev_ip = curr_ip; - if (prev_ip) prev_ip = zstrdup(prev_ip); + if (prev_ip) zfree(prev_ip); + prev_ip = curr_ip; if (curr_ip) { + prev_ip = zstrdup(prev_ip); strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN); myself->ip[NET_IP_STR_LEN-1] = '\0'; } else { From 8442c25722f94ddedcd2068f7bfa10389a2eb79d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:09:38 +0200 Subject: [PATCH 055/616] Introduce writeCommandsDeniedByDiskError(). --- src/server.c | 19 +++++++++++++++++++ src/server.h | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/src/server.c b/src/server.c index c0ea66282..2e0c36014 100644 --- a/src/server.c +++ b/src/server.c @@ -2805,6 +2805,25 @@ int prepareForShutdown(int flags) { /*================================== Commands =============================== */ +/* Sometimes Redis cannot accept write commands because there is a perstence + * error with the RDB or AOF file, and Redis is configured in order to stop + * accepting writes in such situation. This function returns if such a + * condition is active, and the type of the condition. */ +int writeCommandsDeniedByDiskError(void) { + if (server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 && + server.lastbgsave_status == C_ERR) + { + return DISK_ERROR_TYPE_AOF; + } else if (server.aof_state != AOF_OFF && + server.aof_last_write_status == C_ERR) + { + return DISK_ERROR_TYPE_RDB; + } else { + return DISK_ERROR_TYPE_NONE; + } +} + /* Return zero if strings are the same, non-zero if they are not. * The comparison is performed in a way that prevents an attacker to obtain * information about the nature of the strings just monitoring the execution diff --git a/src/server.h b/src/server.h index 21e4c4279..9af80ea32 100644 --- a/src/server.h +++ b/src/server.h @@ -1591,6 +1591,11 @@ void startLoading(FILE *fp); void loadingProgress(off_t pos); void stopLoading(void); +#define DISK_ERROR_TYPE_AOF 1 /* Don't accept writes: AOF errors. */ +#define DISK_ERROR_TYPE_RDB 2 /* Don't accept writes: RDB errors. */ +#define DISK_ERROR_TYPE_NONE 0 /* No problems, we can accept writes. */ +int writeCommandsDeniedByDiskError(void); + /* RDB persistence */ #include "rdb.h" int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi); From 7f155b781cc0abb0e34fbf7214b5eb7b0c64d9ad Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:10:37 +0200 Subject: [PATCH 056/616] Better top comment for writeCommandsDeniedByDiskError(). --- src/server.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 2e0c36014..fbbfba7ff 100644 --- a/src/server.c +++ b/src/server.c @@ -2808,7 +2808,14 @@ int prepareForShutdown(int flags) { /* Sometimes Redis cannot accept write commands because there is a perstence * error with the RDB or AOF file, and Redis is configured in order to stop * accepting writes in such situation. This function returns if such a - * condition is active, and the type of the condition. */ + * condition is active, and the type of the condition. + * + * Function return values: + * + * DISK_ERROR_TYPE_NONE: No problems, we can accept writes. + * DISK_ERROR_TYPE_AOF: Don't accept writes: AOF errors. + * DISK_ERROR_TYPE_RDB: Don't accept writes: RDB errors. + */ int writeCommandsDeniedByDiskError(void) { if (server.stop_writes_on_bgsave_err && server.saveparamslen > 0 && From 0a64aa5ca62392c68c7a4ac3c00cc5c0de9809ab Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:14:24 +0200 Subject: [PATCH 057/616] Fix writeCommandsDeniedByDiskError() inverted return value. --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index fbbfba7ff..28750ab72 100644 --- a/src/server.c +++ b/src/server.c @@ -2821,11 +2821,11 @@ int writeCommandsDeniedByDiskError(void) { server.saveparamslen > 0 && server.lastbgsave_status == C_ERR) { - return DISK_ERROR_TYPE_AOF; + return DISK_ERROR_TYPE_RDB; } else if (server.aof_state != AOF_OFF && server.aof_last_write_status == C_ERR) { - return DISK_ERROR_TYPE_RDB; + return DISK_ERROR_TYPE_AOF; } else { return DISK_ERROR_TYPE_NONE; } From f281d343a7eb085f86f25bb4f87e1ce10f7d9ba2 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 13:16:43 +0200 Subject: [PATCH 058/616] Refactoring: replace low-level checks with writeCommandsDeniedByDiskError(). --- src/scripting.c | 10 +++------- src/server.c | 9 +++------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index b046ea1ff..2732c87fb 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -483,6 +483,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * command marked as non-deterministic was already called in the context * of this script. */ if (cmd->flags & CMD_WRITE) { + int deny_write_type = writeCommandsDeniedByDiskError(); if (server.lua_random_dirty && !server.lua_replicate_commands) { luaPushError(lua, "Write commands not allowed after non deterministic commands. Call redis.replicate_commands() at the start of your script in order to switch to single commands replication mode."); @@ -493,13 +494,8 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { { luaPushError(lua, shared.roslaveerr->ptr); goto cleanup; - } else if ((server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) || - (server.aof_state != AOF_OFF && - server.aof_last_write_status == C_ERR)) - { - if (server.aof_last_write_status == C_OK) { + } else if (deny_write_type != DISK_ERROR_TYPE_NONE) { + if (deny_write_type == DISK_ERROR_TYPE_RDB) { luaPushError(lua, shared.bgsaveerr->ptr); } else { sds aof_write_err = sdscatfmt(sdsempty(), diff --git a/src/server.c b/src/server.c index 28750ab72..66d42ee6c 100644 --- a/src/server.c +++ b/src/server.c @@ -2609,17 +2609,14 @@ int processCommand(client *c) { /* Don't accept write commands if there are problems persisting on disk * and if this is a master instance. */ - if (((server.stop_writes_on_bgsave_err && - server.saveparamslen > 0 && - server.lastbgsave_status == C_ERR) || - (server.aof_state != AOF_OFF && - server.aof_last_write_status == C_ERR)) && + int deny_write_type = writeCommandsDeniedByDiskError(); + if (deny_write_type != DISK_ERROR_TYPE_NONE && server.masterhost == NULL && (c->cmd->flags & CMD_WRITE || c->cmd->proc == pingCommand)) { flagTransaction(c); - if (server.aof_last_write_status == C_OK) + if (deny_write_type == DISK_ERROR_TYPE_RDB) addReply(c, shared.bgsaveerr); else addReplySds(c, From ec67ccc13cadb0ee45c53dd30f68765564cec113 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 16:41:33 +0200 Subject: [PATCH 059/616] Tranfer -> transfer typo fixed. --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index 5d48e3c43..f67073fe6 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -516,7 +516,7 @@ void spopWithCountCommand(client *c) { sdsfree(sdsele); } - /* Tranfer the old set to the client. */ + /* Transfer the old set to the client. */ setTypeIterator *si; si = setTypeInitIterator(set); while((encoding = setTypeNext(si,&sdsele,&llele)) != -1) { From b20e5b3aa38a8fccd73fb6b0e0ba7a7f3b483326 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 17:13:03 +0200 Subject: [PATCH 060/616] Cluster cron announce IP minor refactoring. --- src/cluster.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 7a6a5f782..2f3e298e0 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -3321,14 +3321,17 @@ void clusterCron(void) { int changed = 0; if (prev_ip == NULL && curr_ip != NULL) changed = 1; - if (prev_ip != NULL && curr_ip == NULL) changed = 1; - if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; + else if (prev_ip != NULL && curr_ip == NULL) changed = 1; + else if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1; if (changed) { if (prev_ip) zfree(prev_ip); - prev_ip = curr_ip; + if (curr_ip) { + /* We always take a copy of the previous IP address, by + * duplicating the string. This way later we can check if + * the address really changed. */ prev_ip = zstrdup(prev_ip); strncpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN); myself->ip[NET_IP_STR_LEN-1] = '\0'; From d76d5e06fa05bf33e118ad9460188cf489cdb823 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 17:28:30 +0200 Subject: [PATCH 061/616] Minor improvements to PR #5187. --- tests/integration/psync2.tcl | 10 +++++++--- tests/test_helper.tcl | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 3d9e5527a..b76f36363 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -33,9 +33,8 @@ start_server {} { set cycle 1 while {([clock seconds]-$start_time) < $duration} { - test "PSYNC2: --- CYCLE $cycle ---" { - incr cycle - } + test "PSYNC2: --- CYCLE $cycle ---" {} + incr cycle # Create a random replication layout. # Start with switching master (this simulates a failover). @@ -139,6 +138,11 @@ start_server {} { } assert {$sum == 4} } + + # Limit anyway the maximum number of cycles. This is useful when the + # test is skipped via --only option of the test suite. In that case + # we don't want to see many seconds of this test being just skipped. + if {$cycle > 50} break } test "PSYNC2: Bring the master back again for next test" { diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index ba3dce71c..5d8b149de 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -429,7 +429,7 @@ proc print_help_screen {} { "--single Just execute the specified unit (see next option). this option can be repeated." "--list-tests List all the available test units." "--only Just execute the specified test by test name. this option can be repeated." - "--skiptill Skip all units until (and including) the specified one." + "--skip-till Skip all units until (and including) the specified one." "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." @@ -458,8 +458,7 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set arg2 [lindex $argv [expr $j+2]] lappend ::global_overrides $arg lappend ::global_overrides $arg2 - incr j - incr j + incr j 2 } elseif {$opt eq {--skipfile}} { incr j set fp [open $arg r] @@ -524,6 +523,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +# If --skil-till option was given, we populate the list of single tests +# to run with everything *after* the specified unit. if {$::skip_till != ""} { set skipping 1 foreach t $::all_tests { @@ -540,6 +541,8 @@ if {$::skip_till != ""} { } } +# Override the list of tests with the specific tests we want to run +# in case there was some filter, that is --single or --skip-till options. if {[llength $::single_tests] > 0} { set ::all_tests $::single_tests } From 97f005b23b5d0cd6fbec947ffd4d08734676bb4a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 31 Jul 2018 18:08:52 +0200 Subject: [PATCH 062/616] Streams IDs parsing refactoring. Related to #5184. --- src/t_stream.c | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index f233785ae..72d03b466 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1059,18 +1059,19 @@ int string2ull(const char *s, unsigned long long *value) { * to the client, otherwise C_OK is returned. The ID may be in incomplete * form, just stating the milliseconds time part of the stream. In such a case * the missing part is set according to the value of 'missing_seq' parameter. + * * The IDs "-" and "+" specify respectively the minimum and maximum IDs - * that can be represented. If 'mmid_supp' is set to 0, "-" and "+" will be + * that can be represented. If 'strict' is set to 1, "-" and "+" will be * treated as an invalid ID. * * If 'c' is set to NULL, no reply is sent to the client. */ -int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int mmid_supp) { +int streamGenericParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq, int strict) { char buf[128]; if (sdslen(o->ptr) > sizeof(buf)-1) goto invalid; memcpy(buf,o->ptr,sdslen(o->ptr)+1); - if (!mmid_supp && (buf[0] == '-' || buf[0] == '+') && - buf[1] == '\0') goto invalid; + if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') + goto invalid; /* Handle the "-" and "+" special cases. */ if (buf[0] == '-' && buf[1] == '\0') { @@ -1100,6 +1101,20 @@ invalid: return C_ERR; } +/* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to + * 0, to be used when - and + are accetable IDs. */ +int streamParseIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { + return streamGenericParseIDOrReply(c,o,id,missing_seq,0); +} + +/* Wrapper for streamGenericParseIDOrReply() with 'strict' argument set to + * 1, to be used when we want to return an error if the special IDs + or - + * are provided. */ +int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missing_seq) { + return streamGenericParseIDOrReply(c,o,id,missing_seq,1); +} + + /* XADD key [MAXLEN ] [field value] [field value] ... */ void xaddCommand(client *c) { streamID id; @@ -1137,7 +1152,7 @@ void xaddCommand(client *c) { maxlen_arg_idx = i; } else { /* If we are here is a syntax error or a valid ID. */ - if (streamParseIDOrReply(c,c->argv[i],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[i],&id,0) != C_OK) return; id_given = 1; break; } @@ -1206,8 +1221,8 @@ void xrangeGenericCommand(client *c, int rev) { robj *startarg = rev ? c->argv[3] : c->argv[2]; robj *endarg = rev ? c->argv[2] : c->argv[3]; - if (streamParseIDOrReply(c,startarg,&startid,0,1) == C_ERR) return; - if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX,1) == C_ERR) return; + if (streamParseIDOrReply(c,startarg,&startid,0) == C_ERR) return; + if (streamParseIDOrReply(c,endarg,&endid,UINT64_MAX) == C_ERR) return; /* Parse the COUNT option if any. */ if (c->argc > 4) { @@ -1393,7 +1408,7 @@ void xreadCommand(client *c) { ids[id_idx].seq = UINT64_MAX; continue; } - if (streamParseIDOrReply(c,c->argv[i],ids+id_idx,0,0) != C_OK) + if (streamParseStrictIDOrReply(c,c->argv[i],ids+id_idx,0) != C_OK) goto cleanup; } @@ -1667,7 +1682,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0,0) != C_OK) { + } else if (streamParseStrictIDOrReply(c,c->argv[4],&id,0) != C_OK) { return; } streamCG *cg = streamCreateCG(s,grpname,sdslen(grpname),&id); @@ -1684,7 +1699,7 @@ NULL streamID id; if (!strcmp(c->argv[4]->ptr,"$")) { id = s->last_id; - } else if (streamParseIDOrReply(c,c->argv[4],&id,0,1) != C_OK) { + } else if (streamParseIDOrReply(c,c->argv[4],&id,0) != C_OK) { return; } cg->last_id = id; @@ -1744,7 +1759,7 @@ void xackCommand(client *c) { for (int j = 3; j < c->argc; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL: it will have a reference to the @@ -1791,9 +1806,9 @@ void xpendingCommand(client *c) { if (c->argc >= 6) { if (getLongLongFromObjectOrReply(c,c->argv[5],&count,NULL) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[3],&startid,0,1) == C_ERR) + if (streamParseIDOrReply(c,c->argv[3],&startid,0) == C_ERR) return; - if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX,1) == C_ERR) + if (streamParseIDOrReply(c,c->argv[4],&endid,UINT64_MAX) == C_ERR) return; } @@ -2002,7 +2017,7 @@ void xclaimCommand(client *c) { int j; for (j = 4; j < c->argc; j++) { streamID id; - if (streamParseIDOrReply(NULL,c->argv[j],&id,0,0) != C_OK) break; + if (streamParseStrictIDOrReply(NULL,c->argv[j],&id,0) != C_OK) break; } int last_id_arg = j-1; /* Next time we iterate the IDs we now the range. */ @@ -2061,7 +2076,7 @@ void xclaimCommand(client *c) { for (int j = 5; j <= last_id_arg; j++) { streamID id; unsigned char buf[sizeof(streamID)]; - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; streamEncodeID(buf,&id); /* Lookup the ID in the group PEL. */ @@ -2144,13 +2159,13 @@ void xdelCommand(client *c) { * executed because at some point an invalid ID is parsed. */ streamID id; for (int j = 2; j < c->argc; j++) { - if (streamParseIDOrReply(c,c->argv[j],&id,0,0) != C_OK) return; + if (streamParseStrictIDOrReply(c,c->argv[j],&id,0) != C_OK) return; } /* Actually apply the command. */ int deleted = 0; for (int j = 2; j < c->argc; j++) { - streamParseIDOrReply(c,c->argv[j],&id,0,0); /* Retval already checked. */ + streamParseStrictIDOrReply(c,c->argv[j],&id,0); /* Retval already checked. */ deleted += streamDeleteItem(s,&id); } From 7be529790b5f23c8abbca3a3e493903144fb2131 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 17 Jul 2018 23:57:42 +0800 Subject: [PATCH 063/616] Streams: propagate original MAXLEN argument in XADD context If we rewrite the MAXLEN argument as zero when no trimming was performed, date between master and slave and aof will be inconsistent, because `xtrim maxlen 0` means delete all entries in stream. --- src/t_stream.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 72d03b466..5814aa132 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1186,18 +1186,9 @@ void xaddCommand(client *c) { notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); server.dirty++; - /* Remove older elements if MAXLEN was specified. */ - if (maxlen >= 0) { - if (!streamTrimByLength(s,maxlen,approx_maxlen)) { - /* If no trimming was performed, for instance because approximated - * trimming length was specified, rewrite the MAXLEN argument - * as zero, so that the command is propagated without trimming. */ - robj *zeroobj = createStringObjectFromLongLong(0); - rewriteClientCommandArgument(c,maxlen_arg_idx,zeroobj); - decrRefCount(zeroobj); - } else { - notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); - } + /* Notify xtrim event if needed. */ + if (maxlen >= 0 && streamTrimByLength(s,maxlen,approx_maxlen)) { + notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); } /* Let's rewrite the ID argument with the one actually generated for From c2552025c33a8d2c139ce4519b95b5fbe4797629 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 00:12:24 +0800 Subject: [PATCH 064/616] Streams: XTRIM will return an error if MAXLEN with a count < 0 --- src/t_stream.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 5814aa132..bab5b74ff 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2192,7 +2192,7 @@ void xtrimCommand(client *c) { /* Argument parsing. */ int trim_strategy = TRIM_STRATEGY_NONE; - long long maxlen = 0; /* 0 means no maximum length. */ + long long maxlen = -1; /* If left to -1 no trimming is performed. */ int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so the maxium length is not applied verbatim. */ @@ -2211,6 +2211,11 @@ void xtrimCommand(client *c) { } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; + + if (maxlen < 0) { + addReplyError(c,"The MAXLEN argument must be >= 0."); + return; + } i++; } else { addReply(c,shared.syntaxerr); From 7a22f89e9d7f80ea5f1b429110e4449a375f2177 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 00:24:50 +0800 Subject: [PATCH 065/616] Streams: reset approx_maxlen in every maxlen loop --- src/t_stream.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index bab5b74ff..7533ba7b0 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1135,6 +1135,7 @@ void xaddCommand(client *c) { * creation. */ break; } else if (!strcasecmp(opt,"maxlen") && moreargs) { + approx_maxlen = 0; char *next = c->argv[i+1]->ptr; /* Check for the form MAXLEN ~ . */ if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { @@ -2202,6 +2203,7 @@ void xtrimCommand(client *c) { int moreargs = (c->argc-1) - i; /* Number of additional arguments. */ char *opt = c->argv[i]->ptr; if (!strcasecmp(opt,"maxlen") && moreargs) { + approx_maxlen = 0; trim_strategy = TRIM_STRATEGY_MAXLEN; char *next = c->argv[i+1]->ptr; /* Check for the form MAXLEN ~ . */ From d25a295bc1b4fd7387298efc51b2c4f05f2bb726 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 01:58:14 +0800 Subject: [PATCH 066/616] Streams: propagate specified MAXLEN instead of approximated Slaves and rebooting redis may have different radix tree struct, by different stream* config options. So propagating approximated MAXLEN to AOF/slaves may lead to date inconsistency. --- src/t_stream.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 7533ba7b0..7eaf0c547 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1114,8 +1114,7 @@ int streamParseStrictIDOrReply(client *c, robj *o, streamID *id, uint64_t missin return streamGenericParseIDOrReply(c,o,id,missing_seq,1); } - -/* XADD key [MAXLEN ] [field value] [field value] ... */ +/* XADD key [MAXLEN [~|=] ] [field value] [field value] ... */ void xaddCommand(client *c) { streamID id; int id_given = 0; /* Was an ID different than "*" specified? */ @@ -1141,6 +1140,8 @@ void xaddCommand(client *c) { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { approx_maxlen = 1; i++; + } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') { + i++; } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; @@ -1187,9 +1188,22 @@ void xaddCommand(client *c) { notifyKeyspaceEvent(NOTIFY_STREAM,"xadd",c->argv[1],c->db->id); server.dirty++; - /* Notify xtrim event if needed. */ - if (maxlen >= 0 && streamTrimByLength(s,maxlen,approx_maxlen)) { - notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); + if (maxlen >= 0) { + /* Notify xtrim event if needed. */ + if (streamTrimByLength(s,maxlen,approx_maxlen)) { + notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); + } + + /* Rewrite approximated MAXLEN as specified s->length. */ + if (approx_maxlen) { + robj *maxlen_obj = createStringObjectFromLongLong(s->length); + rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj); + decrRefCount(maxlen_obj); + + robj *equal_obj = createStringObject("=",1); + rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj); + decrRefCount(equal_obj); + } } /* Let's rewrite the ID argument with the one actually generated for @@ -2174,7 +2188,7 @@ void xdelCommand(client *c) { * * List of options: * - * MAXLEN [~] -- Trim so that the stream will be capped at + * MAXLEN [~|=] -- Trim so that the stream will be capped at * the specified length. Use ~ before the * count in order to demand approximated trimming * (like XADD MAXLEN option). @@ -2196,6 +2210,7 @@ void xtrimCommand(client *c) { long long maxlen = -1; /* If left to -1 no trimming is performed. */ int approx_maxlen = 0; /* If 1 only delete whole radix tree nodes, so the maxium length is not applied verbatim. */ + int maxlen_arg_idx = 0; /* Index of the count in MAXLEN, for rewriting. */ /* Parse options. */ int i = 2; /* Start of options. */ @@ -2210,6 +2225,8 @@ void xtrimCommand(client *c) { if (moreargs >= 2 && next[0] == '~' && next[1] == '\0') { approx_maxlen = 1; i++; + } else if (moreargs >= 2 && next[0] == '=' && next[1] == '\0') { + i++; } if (getLongLongFromObjectOrReply(c,c->argv[i+1],&maxlen,NULL) != C_OK) return; @@ -2219,6 +2236,7 @@ void xtrimCommand(client *c) { return; } i++; + maxlen_arg_idx = i; } else { addReply(c,shared.syntaxerr); return; @@ -2239,6 +2257,17 @@ void xtrimCommand(client *c) { signalModifiedKey(c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STREAM,"xtrim",c->argv[1],c->db->id); server.dirty += deleted; + + /* Rewrite approximated MAXLEN as specified s->length. */ + if (approx_maxlen) { + robj *maxlen_obj = createStringObjectFromLongLong(s->length); + rewriteClientCommandArgument(c,maxlen_arg_idx,maxlen_obj); + decrRefCount(maxlen_obj); + + robj *equal_obj = createStringObject("=",1); + rewriteClientCommandArgument(c,maxlen_arg_idx-1,equal_obj); + decrRefCount(equal_obj); + } } addReplyLongLong(c,deleted); } From d7dc6851ff4e814bc288c2c54e7f2a102dfa3526 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 18 Jul 2018 16:55:25 +0800 Subject: [PATCH 067/616] Streams: add test cases for XADD/XTRIM maxlen --- tests/unit/type/stream.tcl | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index 5cf6805d7..2b69a2e9e 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -317,3 +317,49 @@ start_server { assert_equal [r xrevrange teststream2 1234567891245 -] {{1234567891240-0 {key1 value2}} {1234567891230-0 {key1 value1}}} } } + +start_server {tags {"stream"} overrides {appendonly yes}} { + test {XADD with MAXLEN > xlen can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XADD mystream MAXLEN 200 * xitem v + incr j + assert {[r xlen mystream] == $j} + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == $j} + } +} + +start_server {tags {"stream"} overrides {appendonly yes}} { + test {XADD with ~ MAXLEN can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XADD mystream MAXLEN ~ $j * xitem v + incr j + assert {[r xlen mystream] == $j} + r config set stream-node-max-entries 1 + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == $j} + } +} + +start_server {tags {"stream"} overrides {appendonly yes stream-node-max-entries 10}} { + test {XTRIM with ~ MAXLEN can propagate correctly} { + for {set j 0} {$j < 100} {incr j} { + r XADD mystream * xitem v + } + r XTRIM mystream MAXLEN ~ 85 + assert {[r xlen mystream] == 89} + r config set stream-node-max-entries 1 + r debug loadaof + r XADD mystream * xitem v + incr j + assert {[r xlen mystream] == 90} + } +} From 42a385fbc6f4bc86528c094dce930fed72d47386 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:27:56 +0200 Subject: [PATCH 068/616] zsetAdd() refactored adding zslUpdateScore(). --- src/t_zset.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 92abb49d5..06fb61063 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -244,6 +244,23 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { return 0; /* not found */ } +/* Update the score of an elmenent inside the sorted set skiplist. + * Note that the element must exist and must match 'score'. + * This function does not update the score in the hash table side, the + * caller should take care of it. + * + * The function returns the updated element skiplist node pointer. */ +zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { + zskiplistNode *node, *newnode; + serverAssert(zslDelete(zsl,curscore,ele,&node)); + newnode = zslInsert(zsl,newscore,node->ele); + /* We reused the node->ele SDS string, free the node now + * since zslInsert created a new one. */ + node->ele = NULL; + zslFreeNode(node); + return newnode; +} + int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); } @@ -1341,13 +1358,7 @@ int zsetAdd(robj *zobj, double score, sds ele, int *flags, double *newscore) { /* Remove and re-insert when score changes. */ if (score != curscore) { - zskiplistNode *node; - serverAssert(zslDelete(zs->zsl,curscore,ele,&node)); - znode = zslInsert(zs->zsl,score,node->ele); - /* We reused the node->ele SDS string, free the node now - * since zslInsert created a new one. */ - node->ele = NULL; - zslFreeNode(node); + znode = zslUpdateScore(zs->zsl,curscore,ele,score); /* Note that we did not removed the original element from * the hash table representing the sorted set, so we just * update the score. */ From 50fb0cfb1cba6e4d28a199b8b7a27f37e88faa62 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:50:31 +0200 Subject: [PATCH 069/616] Optimize zslUpdateScore() as asked in #5179. --- src/t_zset.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 06fb61063..33e9aa141 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -250,6 +250,49 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { * caller should take care of it. * * The function returns the updated element skiplist node pointer. */ +zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { + zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; + int i; + + x = zsl->header; + for (i = zsl->level-1; i >= 0; i--) { + while (x->level[i].forward && + (x->level[i].forward->score < curscore || + (x->level[i].forward->score == curscore && + sdscmp(x->level[i].forward->ele,ele) < 0))) + { + x = x->level[i].forward; + } + update[i] = x; + } + + /* Jump to our element: note that this function assumes that the + * element with the matching score exists. */ + x = x->level[0].forward; + serverAssert(x && curscore == x->score && sdscmp(x->ele,ele) == 0); + + /* If the node, after the score update, would be still exactly + * at the same position, we can just update the score without + * actually removing and re-inserting the element in the skiplist. */ + if ((x->backward == NULL || x->backward->score <= newscore) && + (x->level[0].forward == NULL || x->level[0].forward->score >= newscore)) + { + x->score = newscore; + return x; + } + + /* No way to reuse the old node: we need to remove and insert a new + * one at a different place. */ + zslDeleteNode(zsl, x, update); + zskiplistNode *newnode = zslInsert(zsl,newscore,x->ele); + /* We reused the old node x->ele SDS string, free the node now + * since zslInsert created a new one. */ + x->ele = NULL; + zslFreeNode(x); + return newnode; +} + +#if 0 zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { zskiplistNode *node, *newnode; serverAssert(zslDelete(zsl,curscore,ele,&node)); @@ -260,6 +303,7 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n zslFreeNode(node); return newnode; } +#endif int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); From f186730634baffc4e89f6ffdf4725b36d1372698 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:50:49 +0200 Subject: [PATCH 070/616] Remove old commented zslUpdateScore() from source. --- src/t_zset.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 33e9aa141..91fe01d76 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -292,19 +292,6 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n return newnode; } -#if 0 -zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { - zskiplistNode *node, *newnode; - serverAssert(zslDelete(zsl,curscore,ele,&node)); - newnode = zslInsert(zsl,newscore,node->ele); - /* We reused the node->ele SDS string, free the node now - * since zslInsert created a new one. */ - node->ele = NULL; - zslFreeNode(node); - return newnode; -} -#endif - int zslValueGteMin(double value, zrangespec *spec) { return spec->minex ? (value > spec->min) : (value >= spec->min); } From 65460646108147e6754493e611180c04caad0693 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:53:06 +0200 Subject: [PATCH 071/616] Explain what's the point of zslUpdateScore() in top comment. --- src/t_zset.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 91fe01d76..16ec715e8 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -249,6 +249,11 @@ int zslDelete(zskiplist *zsl, double score, sds ele, zskiplistNode **node) { * This function does not update the score in the hash table side, the * caller should take care of it. * + * Note that this function attempts to just update the node, in case after + * the score update, the node would be exactly at the same position. + * Otherwise the skiplist is modified by removing and re-adding a new + * element, which is more costly. + * * The function returns the updated element skiplist node pointer. */ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double newscore) { zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; From 6ae26a3db126ad33ae2af43058ff8ca31ff92f32 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 18:54:15 +0200 Subject: [PATCH 072/616] More commenting of zslUpdateScore(). --- src/t_zset.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_zset.c b/src/t_zset.c index 16ec715e8..a794b44c3 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -259,6 +259,8 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x; int i; + /* We need to seek to element to update to start: this is useful anyway, + * we'll have to update or remove it. */ x = zsl->header; for (i = zsl->level-1; i >= 0; i--) { while (x->level[i].forward && From 156f26d0be4449db946761c442ec6c3f79cd25b2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 1 Aug 2018 19:01:40 +0200 Subject: [PATCH 073/616] Fix zslUpdateScore() edge case. When the element new score is the same of prev/next node, the lexicographical order kicks in, so we can safely update the node in place only when the new score is strictly between the adjacent nodes but never equal to one of them. Technically speaking we could do extra checks to make sure that even if the score is the same as one of the adjacent nodes, we can still update on place, but this rarely happens, so probably not a good deal to make it more complex. Related to #5179. --- src/t_zset.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index a794b44c3..db381b592 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -281,8 +281,8 @@ zskiplistNode *zslUpdateScore(zskiplist *zsl, double curscore, sds ele, double n /* If the node, after the score update, would be still exactly * at the same position, we can just update the score without * actually removing and re-inserting the element in the skiplist. */ - if ((x->backward == NULL || x->backward->score <= newscore) && - (x->level[0].forward == NULL || x->level[0].forward->score >= newscore)) + if ((x->backward == NULL || x->backward->score < newscore) && + (x->level[0].forward == NULL || x->level[0].forward->score > newscore)) { x->score = newscore; return x; From d86f233cf34c5acf3f7cb6d2c368690a74f522f9 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 14:14:39 +0200 Subject: [PATCH 074/616] Test: new sorted set skiplist order consistency. This should be able to find new bugs and regressions about the new sorted set update function when ZADD is used to update an element already existing. The test is able to find the bug fixed at 156f26d0 immediately. --- tests/unit/type/zset.tcl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index bebba96d9..cf54ae839 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1185,4 +1185,30 @@ start_server {tags {"zset"}} { stressers ziplist stressers skiplist } + + test {ZSET skiplist order consistency when elements are moved} { + set original_max [lindex [r config get zset-max-ziplist-entries] 1] + r config set zset-max-ziplist-entries 0 + for {set times 0} {$times < 10} {incr times} { + r del zset + for {set j 0} {$j < 1000} {incr j} { + r zadd zset [randomInt 50] ele-[randomInt 10] + } + + # Make sure that element ordering is correct + set prev_element {} + set prev_score -1 + foreach {element score} [r zrange zset 0 -1 WITHSCORES] { + # Assert that elements are in increasing ordering + assert { + $prev_score < $score || + ($prev_score == $score && + [string compare $prev_element $element] == -1) + } + set prev_element $element + set prev_score $score + } + } + r config set zset-max-ziplist-entries $original_max + } } From 0aa7683dbc99ada38d628fbe61e401234bab8c93 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 18:49:49 +0200 Subject: [PATCH 075/616] Test suite: new --stop option. It pauses the test execution once the first failure is found. --- tests/test_helper.tcl | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 5d8b149de..753a0f9b9 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -89,6 +89,7 @@ set ::last_progress [clock seconds] set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 set ::wait_server 0 +set ::stop_on_failure 0 # Set to 1 when we are running in client mode. The Redis test uses a # server-client model to run tests simultaneously. The server instance @@ -302,6 +303,11 @@ proc read_from_test_client fd { puts $err lappend ::failed_tests $err set ::active_clients_task($fd) "(ERR) $data" + if {$::stop_on_failure} { + puts -nonewline "(Test stopped, press enter to continue)" + flush stdout + gets stdin + } } elseif {$status eq {exception}} { puts "\[[colorstr red $status]\]: $data" kill_clients @@ -433,10 +439,11 @@ proc print_help_screen {} { "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." - "--config extra config file argument" - "--skipfile name of a file containing test names that should be skipped (one per line)" - "--dont-clean don't delete redis log files after the run" - "--wait-server wait after server is started (so that you can attach a debugger)" + "--config Extra config file argument" + "--skipfile Name of a file containing test names that should be skipped (one per line)" + "--dont-clean Don't delete redis log files after the run" + "--stop Blocks once the first test fails" + "--wait-server Wait after server is started (so that you can attach a debugger)" "--help Print this help screen." } "\n"] } @@ -511,6 +518,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set ::dont_clean 1 } elseif {$opt eq {--wait-server}} { set ::wait_server 1 + } elseif {$opt eq {--stop}} { + set ::stop_on_failure 1 } elseif {$opt eq {--timeout}} { set ::timeout $arg incr j From 104d7179b654317eb01f507ce067e498fd2c35a9 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 Aug 2018 19:07:17 +0200 Subject: [PATCH 076/616] Test suite: add --loop option. Very useful with --stop in order to catch heisenbugs. --- tests/test_helper.tcl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 753a0f9b9..1a986e2f7 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -90,6 +90,7 @@ set ::active_servers {} ; # Pids of active Redis instances. set ::dont_clean 0 set ::wait_server 0 set ::stop_on_failure 0 +set ::loop 0 # Set to 1 when we are running in client mode. The Redis test uses a # server-client model to run tests simultaneously. The server instance @@ -370,6 +371,9 @@ proc signal_idle_client fd { send_data_packet $fd run [lindex $::all_tests $::next_test] lappend ::active_clients $fd incr ::next_test + if {$::loop && $::next_test == [llength $::all_tests]} { + set ::next_test 0 + } } else { lappend ::idle_clients $fd if {[llength $::active_clients] == 0} { @@ -439,11 +443,12 @@ proc print_help_screen {} { "--clients Number of test clients (default 16)." "--timeout Test timeout in seconds (default 10 min)." "--force-failure Force the execution of a test that always fails." - "--config Extra config file argument" - "--skipfile Name of a file containing test names that should be skipped (one per line)" - "--dont-clean Don't delete redis log files after the run" - "--stop Blocks once the first test fails" - "--wait-server Wait after server is started (so that you can attach a debugger)" + "--config Extra config file argument." + "--skipfile Name of a file containing test names that should be skipped (one per line)." + "--dont-clean Don't delete redis log files after the run." + "--stop Blocks once the first test fails." + "--loop Execute the specified set of tests forever." + "--wait-server Wait after server is started (so that you can attach a debugger)." "--help Print this help screen." } "\n"] } @@ -520,6 +525,8 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set ::wait_server 1 } elseif {$opt eq {--stop}} { set ::stop_on_failure 1 + } elseif {$opt eq {--loop}} { + set ::loop 1 } elseif {$opt eq {--timeout}} { set ::timeout $arg incr j From 97f51864d66105c7bc717d7767e41b63c818f1cc Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 3 Aug 2018 12:46:04 +0200 Subject: [PATCH 077/616] Fix AOF comment to report the current behavior. Realted to #5201. --- src/aof.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index be416ec4e..f8f26bdfe 100644 --- a/src/aof.c +++ b/src/aof.c @@ -798,7 +798,9 @@ int loadAppendOnlyFile(char *filename) { } /* This point can only be reached when EOF is reached without errors. - * If the client is in the middle of a MULTI/EXEC, log error and quit. */ + * If the client is in the middle of a MULTI/EXEC, handle it as it was + * a short read, even if technically the protocol is correct: we want + * to remove the unprocessed tail and continue. */ if (fakeClient->flags & CLIENT_MULTI) goto uxeof; loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ From 7a184d79165ac30311a7fe7d455ad32981b66939 Mon Sep 17 00:00:00 2001 From: shenlongxing Date: Fri, 3 Aug 2018 19:01:15 +0800 Subject: [PATCH 078/616] Fix stream command paras --- src/server.c | 10 +++++----- src/t_stream.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/server.c b/src/server.c index 66d42ee6c..b537ee04a 100644 --- a/src/server.c +++ b/src/server.c @@ -310,14 +310,14 @@ struct redisCommand redisCommandTable[] = { {"xrange",xrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xrevrange",xrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xlen",xlenCommand,2,"rF",0,NULL,1,1,1,0,0}, - {"xread",xreadCommand,-3,"rs",0,xreadGetKeys,1,1,1,0,0}, - {"xreadgroup",xreadCommand,-3,"ws",0,xreadGetKeys,1,1,1,0,0}, + {"xread",xreadCommand,-4,"rs",0,xreadGetKeys,1,1,1,0,0}, + {"xreadgroup",xreadCommand,-7,"ws",0,xreadGetKeys,1,1,1,0,0}, {"xgroup",xgroupCommand,-2,"wm",0,NULL,2,2,1,0,0}, - {"xack",xackCommand,-3,"wF",0,NULL,1,1,1,0,0}, + {"xack",xackCommand,-4,"wF",0,NULL,1,1,1,0,0}, {"xpending",xpendingCommand,-3,"r",0,NULL,1,1,1,0,0}, - {"xclaim",xclaimCommand,-5,"wF",0,NULL,1,1,1,0,0}, + {"xclaim",xclaimCommand,-6,"wF",0,NULL,1,1,1,0,0}, {"xinfo",xinfoCommand,-2,"r",0,NULL,2,2,1,0,0}, - {"xdel",xdelCommand,-2,"wF",0,NULL,1,1,1,0,0}, + {"xdel",xdelCommand,-3,"wF",0,NULL,1,1,1,0,0}, {"xtrim",xtrimCommand,-2,"wF",0,NULL,1,1,1,0,0}, {"post",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"host:",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, diff --git a/src/t_stream.c b/src/t_stream.c index 72d03b466..d31563217 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -2040,12 +2040,12 @@ void xclaimCommand(client *c) { } else if (!strcasecmp(opt,"TIME") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&deliverytime, - "Invalid IDLE option argument for XCLAIM") + "Invalid TIME option argument for XCLAIM") != C_OK) return; } else if (!strcasecmp(opt,"RETRYCOUNT") && moreargs) { j++; if (getLongLongFromObjectOrReply(c,c->argv[j],&retrycount, - "Invalid IDLE option argument for XCLAIM") + "Invalid RETRYCOUNT option argument for XCLAIM") != C_OK) return; } else { addReplyErrorFormat(c,"Unrecognized XCLAIM option '%s'",opt); From 7b773c6eea669883ef3a8b19d4aaf6152020a837 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 2 Aug 2018 14:59:28 +0800 Subject: [PATCH 079/616] AOF: discard if we lost EXEC when loading aof --- src/aof.c | 15 +++++++++++++-- src/multi.c | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/aof.c b/src/aof.c index f8f26bdfe..0c7f28269 100644 --- a/src/aof.c +++ b/src/aof.c @@ -677,6 +677,7 @@ int loadAppendOnlyFile(char *filename) { int old_aof_state = server.aof_state; long loops = 0; off_t valid_up_to = 0; /* Offset of latest well-formed command loaded. */ + off_t valid_before_multi = 0; /* Offset before MULTI command loaded. */ if (fp == NULL) { serverLog(LL_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno)); @@ -781,9 +782,15 @@ int loadAppendOnlyFile(char *filename) { exit(1); } + if (cmd == server.multiCommand) valid_before_multi = valid_up_to; + /* Run the command in the context of a fake client */ fakeClient->cmd = cmd; - cmd->proc(fakeClient); + if (fakeClient->flags & CLIENT_MULTI && fakeClient->cmd->proc != execCommand) { + queueMultiCommand(fakeClient); + } else { + cmd->proc(fakeClient); + } /* The fake client should not have a reply */ serverAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0); @@ -801,7 +808,11 @@ int loadAppendOnlyFile(char *filename) { * If the client is in the middle of a MULTI/EXEC, handle it as it was * a short read, even if technically the protocol is correct: we want * to remove the unprocessed tail and continue. */ - if (fakeClient->flags & CLIENT_MULTI) goto uxeof; + if (fakeClient->flags & CLIENT_MULTI) { + serverLog(LL_WARNING,"!!! Warning: we lost EXEC in the middle of transaction, discard !!!"); + valid_up_to = valid_before_multi; + goto uxeof; + } loaded_ok: /* DB loaded, cleanup and return C_OK to the caller. */ fclose(fp); diff --git a/src/multi.c b/src/multi.c index 112ce0605..8159adcb3 100644 --- a/src/multi.c +++ b/src/multi.c @@ -158,7 +158,7 @@ void execCommand(client *c) { must_propagate = 1; } - call(c,CMD_CALL_FULL); + call(c,server.loading ? CMD_CALL_NONE : CMD_CALL_FULL); /* Commands may alter argc/argv, restore mstate. */ c->mstate.commands[j].argc = c->argc; From 401e63fe599990b739fcc6f89cba262e49bb85d5 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Sat, 4 Aug 2018 01:06:53 +0800 Subject: [PATCH 080/616] Streams: update listpack with new pointer in XDEL --- src/t_stream.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/t_stream.c b/src/t_stream.c index 77fbf4645..c5d3c7f9d 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -724,6 +724,9 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { p = lpNext(lp,p); /* Seek deleted field. */ aux = lpGetInteger(p); lp = lpReplaceInteger(lp,&p,aux+1); + + /* Update the listpack with the new pointer. */ + raxInsert(si->stream->rax,si->ri.key,si->ri.key_len,lp,NULL); } /* Update the number of entries counter. */ From 588b402a7bd1aa7f1c54a367d6d450363a6daae9 Mon Sep 17 00:00:00 2001 From: Jeffrey Lovitz Date: Sun, 12 Aug 2018 12:47:01 -0400 Subject: [PATCH 081/616] CLI Help text loop verifies arg count --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0e8777bd2..247d1f734 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -545,7 +545,7 @@ static void cliIntegrateHelp(void) { ch->params = sdscat(ch->params,"key "); args--; } - while(args--) ch->params = sdscat(ch->params,"arg "); + while(args-- > 0) ch->params = sdscat(ch->params,"arg "); if (entry->element[1]->integer < 0) ch->params = sdscat(ch->params,"...options..."); ch->summary = "Help not available"; From cdda01edefd0de85b6dbc12bccdc90870776796f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 13 Aug 2018 17:36:54 +0300 Subject: [PATCH 082/616] script cache memory in INFO and MEMORY includes both script code and overheads --- src/object.c | 2 +- src/server.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object.c b/src/object.c index 406816140..90ccbc916 100644 --- a/src/object.c +++ b/src/object.c @@ -1017,7 +1017,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mh->aof_buffer = mem; mem_total+=mem; - mem = 0; + mem = server.lua_scripts_mem; mem += dictSize(server.lua_scripts) * sizeof(dictEntry) + dictSlots(server.lua_scripts) * sizeof(dictEntry*); mem += dictSize(server.repl_scriptcache_dict) * sizeof(dictEntry) + diff --git a/src/server.c b/src/server.c index b537ee04a..726147bf1 100644 --- a/src/server.c +++ b/src/server.c @@ -3186,7 +3186,7 @@ sds genRedisInfoString(char *section) { bytesToHuman(peak_hmem,server.stat_peak_memory); bytesToHuman(total_system_hmem,total_system_mem); bytesToHuman(used_memory_lua_hmem,memory_lua); - bytesToHuman(used_memory_scripts_hmem,server.lua_scripts_mem); + bytesToHuman(used_memory_scripts_hmem,mh->lua_caches); bytesToHuman(used_memory_rss_hmem,server.cron_malloc_stats.process_rss); bytesToHuman(maxmemory_hmem,server.maxmemory); @@ -3251,7 +3251,7 @@ sds genRedisInfoString(char *section) { total_system_hmem, memory_lua, used_memory_lua_hmem, - server.lua_scripts_mem, + mh->lua_caches, used_memory_scripts_hmem, dictSize(server.lua_scripts), server.maxmemory, From 59c4b6bf001e6872e6b244e25b78d60042a2d13a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 00:43:36 +0800 Subject: [PATCH 083/616] pipeline: do not sdsrange querybuf unless all commands processed This is an optimization for processing pipeline, we discussed a problem in issue #5229: clients may be paused if we apply `CLIENT PAUSE` command, and then querybuf may grow too large, the cost of memmove in sdsrange after parsing a completed command will be horrible. The optimization is that parsing all commands in queyrbuf , after that we can just call sdsrange only once. --- src/networking.c | 87 ++++++++++++++++++++++++++---------------------- src/server.h | 1 + 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/src/networking.c b/src/networking.c index af7422178..945fdd961 100644 --- a/src/networking.c +++ b/src/networking.c @@ -110,6 +110,7 @@ client *createClient(int fd) { c->fd = fd; c->name = NULL; c->bufpos = 0; + c->qb_pos = 0; c->querybuf = sdsempty(); c->pending_querybuf = sdsempty(); c->querybuf_peak = 0; @@ -1119,11 +1120,11 @@ int processInlineBuffer(client *c) { size_t querylen; /* Search for end of line */ - newline = strchr(c->querybuf,'\n'); + newline = strchr(c->querybuf+c->qb_pos,'\n'); /* Nothing to do without a \r\n */ if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big inline request"); setProtocolError("too big inline request",c,0); } @@ -1131,12 +1132,12 @@ int processInlineBuffer(client *c) { } /* Handle the \r\n case. */ - if (newline && newline != c->querybuf && *(newline-1) == '\r') + if (newline && newline != c->querybuf+c->qb_pos && *(newline-1) == '\r') newline--, linefeed_chars++; /* Split the input buffer up to the \r\n */ - querylen = newline-(c->querybuf); - aux = sdsnewlen(c->querybuf,querylen); + querylen = newline-(c->querybuf+c->qb_pos); + aux = sdsnewlen(c->querybuf+c->qb_pos,querylen); argv = sdssplitargs(aux,&argc); sdsfree(aux); if (argv == NULL) { @@ -1152,7 +1153,8 @@ int processInlineBuffer(client *c) { c->repl_ack_time = server.unixtime; /* Leave data after the first line of the query in the buffer */ - sdsrange(c->querybuf,querylen+linefeed_chars,-1); + sdsrange(c->querybuf,c->qb_pos+querylen+linefeed_chars,-1); + c->qb_pos = 0; /* Setup argv array on client structure */ if (argc) { @@ -1182,10 +1184,10 @@ static void setProtocolError(const char *errstr, client *c, long pos) { /* Sample some protocol to given an idea about what was inside. */ char buf[256]; - if (sdslen(c->querybuf) < PROTO_DUMP_LEN) { - snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%s'", c->querybuf); + if (sdslen(c->querybuf)-c->qb_pos < PROTO_DUMP_LEN) { + snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%s'", c->querybuf+c->qb_pos); } else { - snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%.*s' (... more %zu bytes ...) '%.*s'", PROTO_DUMP_LEN/2, c->querybuf, sdslen(c->querybuf)-PROTO_DUMP_LEN, PROTO_DUMP_LEN/2, c->querybuf+sdslen(c->querybuf)-PROTO_DUMP_LEN/2); + snprintf(buf,sizeof(buf),"Query buffer during protocol error: '%.*s' (... more %zu bytes ...) '%.*s'", PROTO_DUMP_LEN/2, c->querybuf+c->qb_pos, sdslen(c->querybuf)-c->qb_pos-PROTO_DUMP_LEN, PROTO_DUMP_LEN/2, c->querybuf+sdslen(c->querybuf)-PROTO_DUMP_LEN/2); } /* Remove non printable chars. */ @@ -1202,6 +1204,7 @@ static void setProtocolError(const char *errstr, client *c, long pos) { } c->flags |= CLIENT_CLOSE_AFTER_REPLY; sdsrange(c->querybuf,pos,-1); + c->qb_pos -= pos; } /* Process the query buffer for client 'c', setting up the client argument @@ -1217,7 +1220,6 @@ static void setProtocolError(const char *errstr, client *c, long pos) { * to be '*'. Otherwise for inline commands processInlineBuffer() is called. */ int processMultibulkBuffer(client *c) { char *newline = NULL; - long pos = 0; int ok; long long ll; @@ -1226,9 +1228,9 @@ int processMultibulkBuffer(client *c) { serverAssertWithInfo(c,NULL,c->argc == 0); /* Multi bulk length cannot be read without a \r\n */ - newline = strchr(c->querybuf,'\r'); + newline = strchr(c->querybuf+c->qb_pos,'\r'); if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big mbulk count string"); setProtocolError("too big mbulk count string",c,0); } @@ -1236,22 +1238,23 @@ int processMultibulkBuffer(client *c) { } /* Buffer should also contain \n */ - if (newline-(c->querybuf) > ((signed)sdslen(c->querybuf)-2)) + if (newline-(c->querybuf+c->qb_pos) > (ssize_t)(sdslen(c->querybuf)-c->qb_pos-2)) return C_ERR; /* We know for sure there is a whole line since newline != NULL, * so go ahead and find out the multi bulk length. */ - serverAssertWithInfo(c,NULL,c->querybuf[0] == '*'); - ok = string2ll(c->querybuf+1,newline-(c->querybuf+1),&ll); + serverAssertWithInfo(c,NULL,c->querybuf[c->qb_pos] == '*'); + ok = string2ll(c->querybuf+1+c->qb_pos,newline-(c->querybuf+1+c->qb_pos),&ll); if (!ok || ll > 1024*1024) { addReplyError(c,"Protocol error: invalid multibulk length"); - setProtocolError("invalid mbulk count",c,pos); + setProtocolError("invalid mbulk count",c,c->qb_pos); return C_ERR; } - pos = (newline-c->querybuf)+2; + c->qb_pos = (newline-c->querybuf)+2; if (ll <= 0) { - sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; return C_OK; } @@ -1266,9 +1269,9 @@ int processMultibulkBuffer(client *c) { while(c->multibulklen) { /* Read bulk length if unknown */ if (c->bulklen == -1) { - newline = strchr(c->querybuf+pos,'\r'); + newline = strchr(c->querybuf+c->qb_pos,'\r'); if (newline == NULL) { - if (sdslen(c->querybuf) > PROTO_INLINE_MAX_SIZE) { + if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c, "Protocol error: too big bulk count string"); setProtocolError("too big bulk count string",c,0); @@ -1278,25 +1281,25 @@ int processMultibulkBuffer(client *c) { } /* Buffer should also contain \n */ - if (newline-(c->querybuf) > ((signed)sdslen(c->querybuf)-2)) + if (newline-(c->querybuf+c->qb_pos) > (ssize_t)(sdslen(c->querybuf)-c->qb_pos-2)) break; - if (c->querybuf[pos] != '$') { + if (c->querybuf[c->qb_pos] != '$') { addReplyErrorFormat(c, "Protocol error: expected '$', got '%c'", - c->querybuf[pos]); - setProtocolError("expected $ but got something else",c,pos); + c->querybuf[c->qb_pos]); + setProtocolError("expected $ but got something else",c,c->qb_pos); return C_ERR; } - ok = string2ll(c->querybuf+pos+1,newline-(c->querybuf+pos+1),&ll); + ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll); if (!ok || ll < 0 || ll > server.proto_max_bulk_len) { addReplyError(c,"Protocol error: invalid bulk length"); - setProtocolError("invalid bulk length",c,pos); + setProtocolError("invalid bulk length",c,c->qb_pos); return C_ERR; } - pos += newline-(c->querybuf+pos)+2; + c->qb_pos = newline-c->querybuf+2; if (ll >= PROTO_MBULK_BIG_ARG) { size_t qblen; @@ -1304,8 +1307,8 @@ int processMultibulkBuffer(client *c) { * try to make it likely that it will start at c->querybuf * boundary so that we can optimize object creation * avoiding a large copy of data. */ - sdsrange(c->querybuf,pos,-1); - pos = 0; + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; qblen = sdslen(c->querybuf); /* Hint the sds library about the amount of bytes this string is * going to contain. */ @@ -1316,14 +1319,14 @@ int processMultibulkBuffer(client *c) { } /* Read bulk argument */ - if (sdslen(c->querybuf)-pos < (size_t)(c->bulklen+2)) { + if (sdslen(c->querybuf)-c->qb_pos < (size_t)(c->bulklen+2)) { /* Not enough data (+2 == trailing \r\n) */ break; } else { /* Optimization: if the buffer contains JUST our bulk element * instead of creating a new object by *copying* the sds we * just use the current sds string. */ - if (pos == 0 && + if (c->qb_pos == 0 && c->bulklen >= PROTO_MBULK_BIG_ARG && sdslen(c->querybuf) == (size_t)(c->bulklen+2)) { @@ -1333,20 +1336,16 @@ int processMultibulkBuffer(client *c) { * likely... */ c->querybuf = sdsnewlen(SDS_NOINIT,c->bulklen+2); sdsclear(c->querybuf); - pos = 0; } else { c->argv[c->argc++] = - createStringObject(c->querybuf+pos,c->bulklen); - pos += c->bulklen+2; + createStringObject(c->querybuf+c->qb_pos,c->bulklen); + c->qb_pos += c->bulklen+2; } c->bulklen = -1; c->multibulklen--; } } - /* Trim to pos */ - if (pos) sdsrange(c->querybuf,pos,-1); - /* We're done when c->multibulk == 0 */ if (c->multibulklen == 0) return C_OK; @@ -1360,8 +1359,9 @@ int processMultibulkBuffer(client *c) { * pending query buffer, already representing a full command, to process. */ void processInputBuffer(client *c) { server.current_client = c; + /* Keep processing while there is something in the input buffer */ - while(sdslen(c->querybuf)) { + while(c->qb_pos < sdslen(c->querybuf)) { /* Return if clients are paused. */ if (!(c->flags & CLIENT_SLAVE) && clientsArePaused()) break; @@ -1377,7 +1377,7 @@ void processInputBuffer(client *c) { /* Determine request type when unknown. */ if (!c->reqtype) { - if (c->querybuf[0] == '*') { + if (c->querybuf[c->qb_pos] == '*') { c->reqtype = PROTO_REQ_MULTIBULK; } else { c->reqtype = PROTO_REQ_INLINE; @@ -1400,7 +1400,7 @@ void processInputBuffer(client *c) { if (processCommand(c) == C_OK) { if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { /* Update the applied replication offset of our master. */ - c->reploff = c->read_reploff - sdslen(c->querybuf); + c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; } /* Don't reset the client structure for clients blocked in a @@ -1416,6 +1416,13 @@ void processInputBuffer(client *c) { if (server.current_client == NULL) break; } } + + /* Trim to pos */ + if (c->qb_pos) { + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; + } + server.current_client = NULL; } diff --git a/src/server.h b/src/server.h index 186d08250..ce127b587 100644 --- a/src/server.h +++ b/src/server.h @@ -710,6 +710,7 @@ typedef struct client { redisDb *db; /* Pointer to currently SELECTed DB. */ robj *name; /* As set by CLIENT SETNAME. */ sds querybuf; /* Buffer we use to accumulate client queries. */ + size_t qb_pos; /* The position we have read in querybuf. */ sds pending_querybuf; /* If this client is flagged as master, this buffer represents the yet not applied portion of the replication stream that we are receiving from From b250f9d710eb504999a30c202ccda6e26c9f01b2 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 00:57:22 +0800 Subject: [PATCH 084/616] adjust qbuf to 26 in test case for client list --- tests/unit/introspection.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index f6477d9c5..2581eb83d 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1,7 +1,7 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=26 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} test {MONITOR can log executed commands} { set rd [redis_deferring_client] From b1a1667ba21c38da94acf0553bade174904c4e12 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 13:55:30 +0800 Subject: [PATCH 085/616] networking: just return C_OK if multibulk processing saw a <= 0 length. --- src/networking.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index 945fdd961..9e1a3a9ef 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1252,11 +1252,8 @@ int processMultibulkBuffer(client *c) { } c->qb_pos = (newline-c->querybuf)+2; - if (ll <= 0) { - sdsrange(c->querybuf,c->qb_pos,-1); - c->qb_pos = 0; - return C_OK; - } + + if (ll <= 0) return C_OK; c->multibulklen = ll; From 0ca9d1d6f0d5528785b5ab376d82bf7c51a4bfe9 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 14:50:37 +0800 Subject: [PATCH 086/616] networking: just move qb_pos instead of sdsrange in processInlineBuffer --- src/networking.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 9e1a3a9ef..24432339b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1152,9 +1152,8 @@ int processInlineBuffer(client *c) { if (querylen == 0 && c->flags & CLIENT_SLAVE) c->repl_ack_time = server.unixtime; - /* Leave data after the first line of the query in the buffer */ - sdsrange(c->querybuf,c->qb_pos+querylen+linefeed_chars,-1); - c->qb_pos = 0; + /* Move querybuffer position to the next query in the buffer. */ + c->qb_pos += querylen+linefeed_chars; /* Setup argv array on client structure */ if (argc) { From b448b16718b2a6be7ff71241693b3b7cdea266be Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 20:58:58 +0800 Subject: [PATCH 087/616] block: rewrite BRPOPLPUSH as RPOPLPUSH to propagate --- src/server.c | 1 + src/server.h | 2 +- src/t_list.c | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index b537ee04a..cd88d3c2f 100644 --- a/src/server.c +++ b/src/server.c @@ -1491,6 +1491,7 @@ void createSharedObjects(void) { shared.rpop = createStringObject("RPOP",4); shared.lpop = createStringObject("LPOP",4); shared.lpush = createStringObject("LPUSH",5); + shared.rpoplpush = createStringObject("RPOPLPUSH",9); shared.zpopmin = createStringObject("ZPOPMIN",7); shared.zpopmax = createStringObject("ZPOPMAX",7); for (j = 0; j < OBJ_SHARED_INTEGERS; j++) { diff --git a/src/server.h b/src/server.h index 186d08250..05be23c9a 100644 --- a/src/server.h +++ b/src/server.h @@ -781,7 +781,7 @@ struct sharedObjectsStruct { *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink, - *rpop, *lpop, *lpush, *zpopmin, *zpopmax, *emptyscan, + *rpop, *lpop, *lpush, *rpoplpush, *zpopmin, *zpopmax, *emptyscan, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*\r\n" */ diff --git a/src/t_list.c b/src/t_list.c index 1414ff31a..987392e6e 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -596,6 +596,9 @@ void rpoplpushCommand(client *c) { signalModifiedKey(c->db,touchedkey); decrRefCount(touchedkey); server.dirty++; + if (c->lastcmd->proc == brpoplpushCommand) { + rewriteClientCommandVector(c,3,shared.rpoplpush,c->argv[1],c->argv[2]); + } } } From 88d8b45fbe222420808e6e3d7da8dba3675d9a3a Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 14 Aug 2018 20:59:32 +0800 Subject: [PATCH 088/616] block: format code --- src/blocked.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 4a667501f..35c33d1cc 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -269,7 +269,7 @@ void handleClientsBlockedOnKeys(void) { robj *dstkey = receiver->bpop.target; int where = (receiver->lastcmd && receiver->lastcmd->proc == blpopCommand) ? - LIST_HEAD : LIST_TAIL; + LIST_HEAD : LIST_TAIL; robj *value = listTypePop(o,where); if (value) { @@ -285,7 +285,7 @@ void handleClientsBlockedOnKeys(void) { { /* If we failed serving the client we need * to also undo the POP operation. */ - listTypePush(o,value,where); + listTypePush(o,value,where); } if (dstkey) decrRefCount(dstkey); From 6f8633fdf2841b3bd27c1c02a13da1a97e524a12 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 21 Aug 2018 11:46:07 +0300 Subject: [PATCH 089/616] Fix unstable tests on slow machines. Few tests had borderline thresholds that were adjusted. The slave buffers test had two issues, preventing the slave buffer from growing: 1) the slave didn't necessarily go to sleep on time, or woke up too early, now using SIGSTOP to make sure it goes to sleep exactly when we want. 2) the master disconnected the slave on timeout --- tests/unit/dump.tcl | 4 ++-- tests/unit/maxmemory.tcl | 34 ++++++++++++++++++++-------------- tests/unit/memefficiency.tcl | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index 91a4df09e..fa4f53b3b 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -33,7 +33,7 @@ start_server {tags {"dump"}} { set now [clock milliseconds] r restore foo [expr $now+3000] $encoded absttl set ttl [r pttl foo] - assert {$ttl >= 2998 && $ttl <= 3000} + assert {$ttl >= 2990 && $ttl <= 3000} r get foo } {bar} @@ -44,7 +44,7 @@ start_server {tags {"dump"}} { r config set maxmemory-policy allkeys-lru r restore foo 0 $encoded idletime 1000 set idle [r object idletime foo] - assert {$idle >= 1000 && $idle <= 1002} + assert {$idle >= 1000 && $idle <= 1010} r get foo } {bar} diff --git a/tests/unit/maxmemory.tcl b/tests/unit/maxmemory.tcl index 7629fe05e..f6d596ca2 100644 --- a/tests/unit/maxmemory.tcl +++ b/tests/unit/maxmemory.tcl @@ -143,9 +143,11 @@ start_server {tags {"maxmemory"}} { } } -proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { +proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline} { start_server {tags {"maxmemory"}} { start_server {} { + set slave_pid [s process_id] + test "$test_name" { set slave [srv 0 client] set slave_host [srv 0 host] set slave_port [srv 0 port] @@ -158,8 +160,10 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { $master setrange "key:$j" 100000 asdf } + # make sure master doesn't disconnect slave because of timeout + $master config set repl-timeout 300 ;# 5 minutes $master config set maxmemory-policy allkeys-random - $master config set client-output-buffer-limit "slave 100000000 100000000 60" + $master config set client-output-buffer-limit "slave 100000000 100000000 300" $master config set repl-backlog-size [expr {10*1024}] $slave slaveof $master_host $master_port @@ -182,9 +186,9 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { # put the slave to sleep set rd_slave [redis_deferring_client] - $rd_slave debug sleep 300 + exec kill -SIGSTOP $slave_pid - # send some 10mb woth of commands that don't increase the memory usage + # send some 10mb worth of commands that don't increase the memory usage if {$pipeline == 1} { set rd_master [redis_deferring_client -1] for {set k 0} {$k < $cmd_count} {incr k} { @@ -218,19 +222,21 @@ proc test_slave_buffers {cmd_count payload_len limit_memory pipeline} { set delta_no_repl [expr {$killed_used_no_repl - $used_no_repl}] assert {$killed_slave_buf == 0} assert {$delta_no_repl > -50*1024 && $delta_no_repl < 50*1024} ;# 1 byte unaccounted for, with 1M commands will consume some 1MB + + } + # unfreeze slave process (after the 'test' succeeded or failed, but before we attempt to terminate the server + exec kill -SIGCONT $slave_pid } } } -test {slave buffer are counted correctly} { - # we wanna use many small commands, and we don't wanna wait long - # so we need to use a pipeline (redis_deferring_client) - # that may cause query buffer to fill and induce eviction, so we disable it - test_slave_buffers 1000000 10 0 1 -} +# test that slave buffer are counted correctly +# we wanna use many small commands, and we don't wanna wait long +# so we need to use a pipeline (redis_deferring_client) +# that may cause query buffer to fill and induce eviction, so we disable it +test_slave_buffers {slave buffer are counted correctly} 1000000 10 0 1 -test {slave buffer don't induce eviction} { - # test again with fewer (and bigger) commands without pipeline, but with eviction - test_slave_buffers 100000 100 1 0 -} +# test that slave buffer don't induce eviction +# test again with fewer (and bigger) commands without pipeline, but with eviction +test_slave_buffers "slave buffer don't induce eviction" 100000 100 1 0 diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index ec71a36b1..8972d577a 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -201,7 +201,7 @@ start_server {tags {"defrag"}} { assert {$frag < 1.1} # due to high fragmentation, 10hz, and active-defrag-cycle-max set to 75, # we expect max latency to be not much higher than 75ms - assert {$max_latency <= 80} + assert {$max_latency <= 120} } # verify the data isn't corrupted or changed set newdigest [r debug digest] From 1091419a0fa4f01a5f4b5a6ad0c7794918603df5 Mon Sep 17 00:00:00 2001 From: "dejun.xdj" Date: Wed, 22 Aug 2018 18:07:02 +0800 Subject: [PATCH 090/616] Revise the comments of latency command. --- src/latency.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/latency.c b/src/latency.c index e8d2af306..d89c48db1 100644 --- a/src/latency.c +++ b/src/latency.c @@ -560,10 +560,11 @@ sds latencyCommandGenSparkeline(char *event, struct latencyTimeSeries *ts) { /* LATENCY command implementations. * - * LATENCY SAMPLES: return time-latency samples for the specified event. + * LATENCY HISTORY: return time-latency samples for the specified event. * LATENCY LATEST: return the latest latency for all the events classes. * LATENCY DOCTOR: returns an human readable analysis of instance latency. * LATENCY GRAPH: provide an ASCII graph of the latency of the specified event. + * LATENCY RESET: reset data of a specified event or all the data if no event provided. */ void latencyCommand(client *c) { struct latencyTimeSeries *ts; From 7e6a4d3f4f67e11cea982faded01bfee2aa12358 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 23 Aug 2018 12:21:23 +0800 Subject: [PATCH 091/616] networking: make setProtocolError simple and clear Function setProtocolError just records proctocol error details in server log, set client as CLIENT_CLOSE_AFTER_REPLY. It doesn't care about querybuf sdsrange, because we will do it after procotol parsing. --- src/networking.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/networking.c b/src/networking.c index 24432339b..58248ced0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -33,7 +33,7 @@ #include #include -static void setProtocolError(const char *errstr, client *c, long pos); +static void setProtocolError(const char *errstr, client *c); /* Return the size consumed from the allocator, for the specified SDS string, * including internal fragmentation. This function is used in order to compute @@ -1126,7 +1126,7 @@ int processInlineBuffer(client *c) { if (newline == NULL) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big inline request"); - setProtocolError("too big inline request",c,0); + setProtocolError("too big inline request",c); } return C_ERR; } @@ -1142,7 +1142,7 @@ int processInlineBuffer(client *c) { sdsfree(aux); if (argv == NULL) { addReplyError(c,"Protocol error: unbalanced quotes in request"); - setProtocolError("unbalanced quotes in inline request",c,0); + setProtocolError("unbalanced quotes in inline request",c); return C_ERR; } @@ -1174,10 +1174,10 @@ int processInlineBuffer(client *c) { return C_OK; } -/* Helper function. Trims query buffer to make the function that processes - * multi bulk requests idempotent. */ +/* Helper function. Record protocol erro details in server log, + * and set the client as CLIENT_CLOSE_AFTER_REPLY. */ #define PROTO_DUMP_LEN 128 -static void setProtocolError(const char *errstr, client *c, long pos) { +static void setProtocolError(const char *errstr, client *c) { if (server.verbosity <= LL_VERBOSE) { sds client = catClientInfoString(sdsempty(),c); @@ -1202,8 +1202,6 @@ static void setProtocolError(const char *errstr, client *c, long pos) { sdsfree(client); } c->flags |= CLIENT_CLOSE_AFTER_REPLY; - sdsrange(c->querybuf,pos,-1); - c->qb_pos -= pos; } /* Process the query buffer for client 'c', setting up the client argument @@ -1231,7 +1229,7 @@ int processMultibulkBuffer(client *c) { if (newline == NULL) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c,"Protocol error: too big mbulk count string"); - setProtocolError("too big mbulk count string",c,0); + setProtocolError("too big mbulk count string",c); } return C_ERR; } @@ -1246,7 +1244,7 @@ int processMultibulkBuffer(client *c) { ok = string2ll(c->querybuf+1+c->qb_pos,newline-(c->querybuf+1+c->qb_pos),&ll); if (!ok || ll > 1024*1024) { addReplyError(c,"Protocol error: invalid multibulk length"); - setProtocolError("invalid mbulk count",c,c->qb_pos); + setProtocolError("invalid mbulk count",c); return C_ERR; } @@ -1270,7 +1268,7 @@ int processMultibulkBuffer(client *c) { if (sdslen(c->querybuf)-c->qb_pos > PROTO_INLINE_MAX_SIZE) { addReplyError(c, "Protocol error: too big bulk count string"); - setProtocolError("too big bulk count string",c,0); + setProtocolError("too big bulk count string",c); return C_ERR; } break; @@ -1284,14 +1282,14 @@ int processMultibulkBuffer(client *c) { addReplyErrorFormat(c, "Protocol error: expected '$', got '%c'", c->querybuf[c->qb_pos]); - setProtocolError("expected $ but got something else",c,c->qb_pos); + setProtocolError("expected $ but got something else",c); return C_ERR; } ok = string2ll(c->querybuf+c->qb_pos+1,newline-(c->querybuf+c->qb_pos+1),&ll); if (!ok || ll < 0 || ll > server.proto_max_bulk_len) { addReplyError(c,"Protocol error: invalid bulk length"); - setProtocolError("invalid bulk length",c,c->qb_pos); + setProtocolError("invalid bulk length",c); return C_ERR; } From c2f52c17528c4c23f58d6fe5fa0369d6f44ceaa6 Mon Sep 17 00:00:00 2001 From: Chris Lamb Date: Sun, 26 Aug 2018 14:45:39 +0200 Subject: [PATCH 092/616] Correct "did not received" -> "did not receive" typos/grammar. --- src/scripting.c | 2 +- src/sentinel.c | 6 +++--- tests/sentinel/tests/00-base.tcl | 4 ++-- tests/sentinel/tests/01-conf-update.tcl | 4 ++-- tests/sentinel/tests/02-slaves-reconf.tcl | 2 +- tests/sentinel/tests/05-manual.tcl | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 2732c87fb..9afc08a94 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1725,7 +1725,7 @@ int ldbRemoveChild(pid_t pid) { return 0; } -/* Return the number of children we still did not received termination +/* Return the number of children we still did not receive termination * acknowledge via wait() in the parent process. */ int ldbPendingChildren(void) { return listLength(ldb.children); diff --git a/src/sentinel.c b/src/sentinel.c index 76e0eb750..b608fc693 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2628,7 +2628,7 @@ int sentinelSendPing(sentinelRedisInstance *ri) { ri->link->last_ping_time = mstime(); /* We update the active ping time only if we received the pong for * the previous ping, otherwise we are technically waiting since the - * first ping that did not received a reply. */ + * first ping that did not receive a reply. */ if (ri->link->act_ping_time == 0) ri->link->act_ping_time = ri->link->last_ping_time; return 1; @@ -3569,7 +3569,7 @@ void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) { (mstime() - ri->link->cc_conn_time) > SENTINEL_MIN_LINK_RECONNECT_PERIOD && ri->link->act_ping_time != 0 && /* There is a pending ping... */ - /* The pending ping is delayed, and we did not received + /* The pending ping is delayed, and we did not receive * error replies as well. */ (mstime() - ri->link->act_ping_time) > (ri->down_after_period/2) && (mstime() - ri->link->last_pong_time) > (ri->down_after_period/2)) @@ -3725,7 +3725,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f * * 1) We believe it is down, or there is a failover in progress. * 2) Sentinel is connected. - * 3) We did not received the info within SENTINEL_ASK_PERIOD ms. */ + * 3) We did not receive the info within SENTINEL_ASK_PERIOD ms. */ if ((master->flags & SRI_S_DOWN) == 0) continue; if (ri->link->disconnected) continue; if (!(flags & SENTINEL_ASK_FORCED) && diff --git a/tests/sentinel/tests/00-base.tcl b/tests/sentinel/tests/00-base.tcl index a79d0c371..7fb1a8bef 100644 --- a/tests/sentinel/tests/00-base.tcl +++ b/tests/sentinel/tests/00-base.tcl @@ -17,7 +17,7 @@ test "Basic failover works if the master is down" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id @@ -108,7 +108,7 @@ test "Failover works if we configure for absolute agreement" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id diff --git a/tests/sentinel/tests/01-conf-update.tcl b/tests/sentinel/tests/01-conf-update.tcl index 4998104d2..d45b1b08e 100644 --- a/tests/sentinel/tests/01-conf-update.tcl +++ b/tests/sentinel/tests/01-conf-update.tcl @@ -16,7 +16,7 @@ test "We can failover with Sentinel 1 crashed" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "Sentinel $id did not received failover info" + fail "Sentinel $id did not receive failover info" } } } @@ -30,7 +30,7 @@ test "After Sentinel 1 is restarted, its config gets updated" { wait_for_condition 1000 50 { [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "Restarted Sentinel did not received failover info" + fail "Restarted Sentinel did not receive failover info" } } diff --git a/tests/sentinel/tests/02-slaves-reconf.tcl b/tests/sentinel/tests/02-slaves-reconf.tcl index fa15d2efd..28964c968 100644 --- a/tests/sentinel/tests/02-slaves-reconf.tcl +++ b/tests/sentinel/tests/02-slaves-reconf.tcl @@ -36,7 +36,7 @@ proc 02_crash_and_failover {} { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } restart_instance redis $master_id diff --git a/tests/sentinel/tests/05-manual.tcl b/tests/sentinel/tests/05-manual.tcl index 5214fdce1..ed568aa03 100644 --- a/tests/sentinel/tests/05-manual.tcl +++ b/tests/sentinel/tests/05-manual.tcl @@ -12,7 +12,7 @@ test "Manual failover works" { wait_for_condition 1000 50 { [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port } else { - fail "At least one Sentinel did not received failover info" + fail "At least one Sentinel did not receive failover info" } } set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] From 55a0a19cf6f9946897f416680cdb4697ba0feccf Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 27 Aug 2018 12:07:24 +0800 Subject: [PATCH 093/616] remove duplicate bind in sentinel.conf --- sentinel.conf | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sentinel.conf b/sentinel.conf index 551defef9..b7fe7bb63 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -30,16 +30,6 @@ daemonize no # location here. pidfile /var/run/redis-sentinel.pid -# By default Redis Sentinel listens for connections from all the network -# interfaces available on the server. It is possible to listen to just one or -# multiple interfaces using the "bind" configuration directive, followed by one -# or more IP addresses. -# -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 - # Specify the log file name. Also the empty string can be used to force # Sentinel to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null From 727382c67c7ec5db81573b1492b20924127c6c04 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:14:10 +0200 Subject: [PATCH 094/616] Fix build errors caused by #2358. --- src/config.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/config.c b/src/config.c index 0bf2f5f41..7bd9592b2 100644 --- a/src/config.c +++ b/src/config.c @@ -887,17 +887,6 @@ void configSetCommand(client *c) { } config_set_special_field("masterauth") { zfree(server.masterauth); server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; - } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { - if (getLongLongFromObject(o,&ll) == REDIS_ERR || - ll < 0) goto badfmt; - server.maxmemory = ll; - if (server.maxmemory) { - if (server.maxmemory < zmalloc_used_memory()) { - redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); - } - freeMemoryIfNeeded(); - } - } else if (!strcasecmp(c->argv[2]->ptr,"maxclients")) { } config_set_special_field("cluster-announce-ip") { zfree(server.cluster_announce_ip); server.cluster_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; From 82121055eb79ec350dc072ef4aa88990c6e037d7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:17:14 +0200 Subject: [PATCH 095/616] Re-apply rebased #2358. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 7bd9592b2..b15cb6d77 100644 --- a/src/config.c +++ b/src/config.c @@ -1181,7 +1181,7 @@ void configSetCommand(client *c) { } config_set_memory_field("maxmemory",server.maxmemory) { if (server.maxmemory) { if (server.maxmemory < zmalloc_used_memory()) { - serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); + serverLog(LL_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in key eviction and/or the inability to accept new write commands depending on the maxmemory-policy."); } freeMemoryIfNeeded(); } From b36fbe9c042d0dd97d8b49d42a7a9883b595adfd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 11:57:44 +0200 Subject: [PATCH 096/616] Better variable meaning in processCommand(). --- src/server.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index cd88d3c2f..aeb30cdf1 100644 --- a/src/server.c +++ b/src/server.c @@ -2594,14 +2594,14 @@ int processCommand(client *c) { * keys in the dataset). If there are not the only thing we can do * is returning an error. */ if (server.maxmemory) { - int retval = freeMemoryIfNeeded(); + int out_of_memory = freeMemoryIfNeeded() == C_ERR; /* freeMemoryIfNeeded may flush slave output buffers. This may result * into a slave, that may be the active client, to be freed. */ if (server.current_client == NULL) return C_ERR; /* It was impossible to free enough memory, and the command the client * is trying to execute is denied during OOM conditions? Error. */ - if ((c->cmd->flags & CMD_DENYOOM) && retval == C_ERR) { + if ((c->cmd->flags & CMD_DENYOOM) && out_of_memory) { flagTransaction(c); addReply(c, shared.oomerr); return C_OK; From d85194c4e11383fedfd9b73361ab1aa04569df91 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:09:08 +0200 Subject: [PATCH 097/616] Introduce repl_slave_ignore_maxmemory flag internally. Note: this breaks backward compatibility with Redis 4, since now slaves by default are exact copies of masters and do not try to evict keys independently. --- src/evict.c | 4 ++++ src/server.c | 1 + src/server.h | 2 ++ 3 files changed, 7 insertions(+) diff --git a/src/evict.c b/src/evict.c index ecc25dd8e..cdb49a551 100644 --- a/src/evict.c +++ b/src/evict.c @@ -444,6 +444,10 @@ int getMaxmemoryState(size_t *total, size_t *logical, size_t *tofree, float *lev * Otehrwise if we are over the memory limit, but not enough memory * was freed to return back under the limit, the function returns C_ERR. */ int freeMemoryIfNeeded(void) { + /* By default slaves should ignore maxmemory and just be masters excat + * copies. */ + if (server.masterhost && server.repl_slave_ignore_maxmemory) return C_OK; + size_t mem_reported, mem_tofree, mem_freed; mstime_t latency, eviction_latency; long long delta; diff --git a/src/server.c b/src/server.c index aeb30cdf1..4aa36c287 100644 --- a/src/server.c +++ b/src/server.c @@ -1649,6 +1649,7 @@ void initServerConfig(void) { server.repl_syncio_timeout = CONFIG_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA; server.repl_slave_ro = CONFIG_DEFAULT_SLAVE_READ_ONLY; + server.repl_slave_ignore_maxmemory = CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY; server.repl_slave_lazy_flush = CONFIG_DEFAULT_SLAVE_LAZY_FLUSH; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.repl_disable_tcp_nodelay = CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY; diff --git a/src/server.h b/src/server.h index 2e1f7c9ce..4fb8e383c 100644 --- a/src/server.h +++ b/src/server.h @@ -131,6 +131,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY 5 #define CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA 1 #define CONFIG_DEFAULT_SLAVE_READ_ONLY 1 +#define CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY 1 #define CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP NULL #define CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT 0 #define CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 @@ -1157,6 +1158,7 @@ struct redisServer { time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */ int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ + int repl_slave_ignore_maxmemory; /* If true slaves do not evict. */ time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ From e2c6f9fc60ce27476390a6f3b3ae810079a0026d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:27:17 +0200 Subject: [PATCH 098/616] Make slave-ignore-maxmemory configurable. --- src/config.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/config.c b/src/config.c index b15cb6d77..1a64d25ed 100644 --- a/src/config.c +++ b/src/config.c @@ -399,6 +399,10 @@ void loadServerConfigFromString(char *config) { if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-ignore-maxmemory") && argc == 2) { + if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -1047,6 +1051,8 @@ void configSetCommand(client *c) { "slave-serve-stale-data",server.repl_serve_stale_data) { } config_set_bool_field( "slave-read-only",server.repl_slave_ro) { + } config_set_bool_field( + "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { } config_set_bool_field( "activerehashing",server.activerehashing) { } config_set_bool_field( @@ -1353,6 +1359,8 @@ void configGetCommand(client *c) { server.repl_serve_stale_data); config_get_bool_field("slave-read-only", server.repl_slave_ro); + config_get_bool_field("slave-ignore-maxmemory", + server.repl_slave_ignore_maxmemory); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); @@ -2049,6 +2057,7 @@ int rewriteConfig(char *path) { rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); + rewriteConfigYesNoOption(state,"slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); From 372cae9b9ea30ece68babeaf3feacd9373bfc709 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 Aug 2018 12:34:29 +0200 Subject: [PATCH 099/616] Document slave-ignore-maxmemory in redis.conf. --- redis.conf | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/redis.conf b/redis.conf index 5cbc74bbd..886c64f21 100644 --- a/redis.conf +++ b/redis.conf @@ -602,6 +602,26 @@ slave-priority 100 # # maxmemory-samples 5 +# Starting from Redis 5, by default a slave will ignore its maxmemory setting +# (unless it is promoted to master after a failover or manually). It means +# that the eviction of keys will be just handled by the master, sending the +# DEL commands to the slave as keys evict in the master side. +# +# This behavior ensures that masters and slaves stay consistent, and is usually +# what you want, however if your slave is writable, or you want the slave to have +# a different memory setting, and you are sure all the writes performed to the +# slave are idempotent, then you may change this default (but be sure to understand +# what you are doing). +# +# Note that since the slave by default does not evict, it may end using more +# memory than the one set via maxmemory (there are certain buffers that may +# be larger on the slave, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your slaves and make sure they have enough +# memory to never hit a real out-of-memory condition before the master hits +# the configured maxmemory setting. +# +# slave-ingore-maxmemory yes + ############################# LAZY FREEING #################################### # Redis has two primitives to delete keys. One is called DEL and is a blocking From ca3d37a4fe68132d709bb51ba2b74586c47ee3be Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 10 Apr 2018 20:41:21 +0800 Subject: [PATCH 100/616] some commands' flags should be set correctly, issue #4834 --- src/server.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/server.c b/src/server.c index 7f51778d7..79227142e 100644 --- a/src/server.c +++ b/src/server.c @@ -210,7 +210,7 @@ struct redisCommand redisCommandTable[] = { {"hstrlen",hstrlenCommand,3,"rF",0,NULL,1,1,1,0,0}, {"hkeys",hkeysCommand,2,"rS",0,NULL,1,1,1,0,0}, {"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0}, - {"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0}, + {"hgetall",hgetallCommand,2,"rR",0,NULL,1,1,1,0,0}, {"hexists",hexistsCommand,3,"rF",0,NULL,1,1,1,0,0}, {"hscan",hscanCommand,-3,"rR",0,NULL,1,1,1,0,0}, {"incrby",incrbyCommand,3,"wmF",0,NULL,1,1,1,0,0}, @@ -236,9 +236,9 @@ struct redisCommand redisCommandTable[] = { {"ping",pingCommand,-1,"tF",0,NULL,0,0,0,0,0}, {"echo",echoCommand,2,"F",0,NULL,0,0,0,0,0}, {"save",saveCommand,1,"as",0,NULL,0,0,0,0,0}, - {"bgsave",bgsaveCommand,-1,"a",0,NULL,0,0,0,0,0}, - {"bgrewriteaof",bgrewriteaofCommand,1,"a",0,NULL,0,0,0,0,0}, - {"shutdown",shutdownCommand,-1,"alt",0,NULL,0,0,0,0,0}, + {"bgsave",bgsaveCommand,-1,"as",0,NULL,0,0,0,0,0}, + {"bgrewriteaof",bgrewriteaofCommand,1,"as",0,NULL,0,0,0,0,0}, + {"shutdown",shutdownCommand,-1,"aslt",0,NULL,0,0,0,0,0}, {"lastsave",lastsaveCommand,1,"RF",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"rF",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"sF",0,NULL,0,0,0,0,0}, @@ -250,16 +250,16 @@ struct redisCommand redisCommandTable[] = { {"flushdb",flushdbCommand,-1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,-1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wm",0,sortGetKeys,1,1,1,0,0}, - {"info",infoCommand,-1,"lt",0,NULL,0,0,0,0,0}, + {"info",infoCommand,-1,"lts",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"as",0,NULL,0,0,0,0,0}, - {"ttl",ttlCommand,2,"rF",0,NULL,1,1,1,0,0}, + {"ttl",ttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, - {"pttl",pttlCommand,2,"rF",0,NULL,1,1,1,0,0}, + {"pttl",pttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"wF",0,NULL,1,1,1,0,0}, {"slaveof",slaveofCommand,3,"ast",0,NULL,0,0,0,0,0}, {"role",roleCommand,1,"lst",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, - {"config",configCommand,-2,"lat",0,NULL,0,0,0,0,0}, + {"config",configCommand,-2,"last",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"pslt",0,NULL,0,0,0,0,0}, {"unsubscribe",unsubscribeCommand,-1,"pslt",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"pslt",0,NULL,0,0,0,0,0}, @@ -271,24 +271,24 @@ struct redisCommand redisCommandTable[] = { {"cluster",clusterCommand,-2,"a",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,-4,"wm",0,NULL,1,1,1,0,0}, {"restore-asking",restoreCommand,-4,"wmk",0,NULL,1,1,1,0,0}, - {"migrate",migrateCommand,-6,"w",0,migrateGetKeys,0,0,0,0,0}, + {"migrate",migrateCommand,-6,"wR",0,migrateGetKeys,0,0,0,0,0}, {"asking",askingCommand,1,"F",0,NULL,0,0,0,0,0}, {"readonly",readonlyCommand,1,"F",0,NULL,0,0,0,0,0}, {"readwrite",readwriteCommand,1,"F",0,NULL,0,0,0,0,0}, - {"dump",dumpCommand,2,"r",0,NULL,1,1,1,0,0}, - {"object",objectCommand,-2,"r",0,NULL,2,2,1,0,0}, - {"memory",memoryCommand,-2,"r",0,NULL,0,0,0,0,0}, + {"dump",dumpCommand,2,"rR",0,NULL,1,1,1,0,0}, + {"object",objectCommand,-2,"rR",0,NULL,2,2,1,0,0}, + {"memory",memoryCommand,-2,"rs",0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,"as",0,NULL,0,0,0,0,0}, {"eval",evalCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, {"evalsha",evalShaCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, - {"slowlog",slowlogCommand,-2,"a",0,NULL,0,0,0,0,0}, + {"slowlog",slowlogCommand,-2,"aR",0,NULL,0,0,0,0,0}, {"script",scriptCommand,-2,"s",0,NULL,0,0,0,0,0}, {"time",timeCommand,1,"RF",0,NULL,0,0,0,0,0}, {"bitop",bitopCommand,-4,"wm",0,NULL,2,-1,1,0,0}, {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0}, {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0}, {"wait",waitCommand,3,"s",0,NULL,0,0,0,0,0}, - {"command",commandCommand,0,"lt",0,NULL,0,0,0,0,0}, + {"command",commandCommand,0,"lts",0,NULL,0,0,0,0,0}, {"geoadd",geoaddCommand,-5,"wm",0,NULL,1,1,1,0,0}, {"georadius",georadiusCommand,-6,"w",0,georadiusGetKeys,1,1,1,0,0}, {"georadius_ro",georadiusroCommand,-6,"r",0,georadiusGetKeys,1,1,1,0,0}, From 5e47f656dfa5474eb37925fdf05a7b27123f4a66 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Wed, 29 Aug 2018 18:23:05 +0800 Subject: [PATCH 101/616] Supplement to PR #4835, just take info/memory/command as random commands --- src/server.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index c8f9ef021..55bcb4dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -254,7 +254,7 @@ struct redisCommand redisCommandTable[] = { {"flushdb",flushdbCommand,-1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,-1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wm",0,sortGetKeys,1,1,1,0,0}, - {"info",infoCommand,-1,"lts",0,NULL,0,0,0,0,0}, + {"info",infoCommand,-1,"ltR",0,NULL,0,0,0,0,0}, {"monitor",monitorCommand,1,"as",0,NULL,0,0,0,0,0}, {"ttl",ttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, @@ -281,7 +281,7 @@ struct redisCommand redisCommandTable[] = { {"readwrite",readwriteCommand,1,"F",0,NULL,0,0,0,0,0}, {"dump",dumpCommand,2,"rR",0,NULL,1,1,1,0,0}, {"object",objectCommand,-2,"rR",0,NULL,2,2,1,0,0}, - {"memory",memoryCommand,-2,"rs",0,NULL,0,0,0,0,0}, + {"memory",memoryCommand,-2,"rR",0,NULL,0,0,0,0,0}, {"client",clientCommand,-2,"as",0,NULL,0,0,0,0,0}, {"eval",evalCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, {"evalsha",evalShaCommand,-3,"s",0,evalGetKeys,0,0,0,0,0}, @@ -292,7 +292,7 @@ struct redisCommand redisCommandTable[] = { {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0}, {"bitpos",bitposCommand,-3,"r",0,NULL,1,1,1,0,0}, {"wait",waitCommand,3,"s",0,NULL,0,0,0,0,0}, - {"command",commandCommand,0,"lts",0,NULL,0,0,0,0,0}, + {"command",commandCommand,0,"ltR",0,NULL,0,0,0,0,0}, {"geoadd",geoaddCommand,-5,"wm",0,NULL,1,1,1,0,0}, {"georadius",georadiusCommand,-6,"w",0,georadiusGetKeys,1,1,1,0,0}, {"georadius_ro",georadiusroCommand,-6,"r",0,georadiusGetKeys,1,1,1,0,0}, From 179dfb60754f8fb5c9dfa934d9384707b77b9c86 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Fri, 31 Aug 2018 11:49:27 +0800 Subject: [PATCH 102/616] networking: fix unexpected negative or zero readlen To avoid copying buffers to create a large Redis Object which exceeding PROTO_IOBUF_LEN 32KB, we just read the remaining data we need, which may less than PROTO_IOBUF_LEN. But the remaining len may be zero, if the bulklen+2 equals sdslen(c->querybuf), in client pause context. For example: Time1: python >>> import os, socket >>> server="127.0.0.1" >>> port=6379 >>> data1="*3\r\n$3\r\nset\r\n$1\r\na\r\n$33000\r\n" >>> data2="".join("x" for _ in range(33000)) + "\r\n" >>> data3="\n\n" >>> s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) >>> s.settimeout(10) >>> s.connect((server, port)) >>> s.send(data1) 28 Time2: redis-cli client pause 10000 Time3: >>> s.send(data2) 33002 >>> s.send(data3) 2 >>> s.send(data3) Traceback (most recent call last): File "", line 1, in socket.error: [Errno 104] Connection reset by peer To fix that, we should check if remaining is greater than zero. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 58248ced0..17d82d1a2 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1439,7 +1439,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { { ssize_t remaining = (size_t)(c->bulklen+2)-sdslen(c->querybuf); - if (remaining < readlen) readlen = remaining; + if (remaining > 0 && remaining < readlen) readlen = remaining; } qblen = sdslen(c->querybuf); From 15334f5355b9df4a9fe77456d6c66eb3d6d2d2c5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 Aug 2018 18:14:46 +0200 Subject: [PATCH 103/616] Allow scripts to timeout on slaves as well. See reasoning in #5297. --- src/scripting.c | 12 +++++++++--- src/server.c | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 9afc08a94..c3ee0f971 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1222,12 +1222,18 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { /* This is the Lua script "count" hook that we use to detect scripts timeout. */ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed; + long long elapsed = mstime() - server.lua_time_start; UNUSED(ar); UNUSED(lua); - elapsed = mstime() - server.lua_time_start; - if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { + /* The conditions to timeout are: + * 1. The caller is not our master. + * 2. The timeout was reached. + * 3. We are not already timed out. */ + if (!(server.lua_caller->flags & CLIENT_MASTER) && + elapsed >= server.lua_time_limit && + server.lua_timedout == 0) + { serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others diff --git a/src/server.c b/src/server.c index 55bcb4dc2..1afc9a802 100644 --- a/src/server.c +++ b/src/server.c @@ -2683,6 +2683,7 @@ int processCommand(client *c) { /* Lua script too slow? Only allow a limited number of commands. */ if (server.lua_timedout && + !(c->flags & CLIENT_MASTER) && c->cmd->proc != authCommand && c->cmd->proc != replconfCommand && !(c->cmd->proc == shutdownCommand && From c3d287793d90a2737aac9e6565e2eabf03866ce1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Aug 2018 11:46:42 +0200 Subject: [PATCH 104/616] Allow scripts to timeout even if from the master instance. However the master scripts will be impossible to kill. Related to #5297. --- src/scripting.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index c3ee0f971..0ef8d2a6f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1226,14 +1226,9 @@ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { UNUSED(ar); UNUSED(lua); - /* The conditions to timeout are: - * 1. The caller is not our master. - * 2. The timeout was reached. - * 3. We are not already timed out. */ - if (!(server.lua_caller->flags & CLIENT_MASTER) && - elapsed >= server.lua_time_limit && - server.lua_timedout == 0) - { + /* Set the timeout condition if not already set and the maximum + * execution time was reached. */ + if (elapsed >= server.lua_time_limit && server.lua_timedout == 0) { serverLog(LL_WARNING,"Lua slow script detected: still in execution after %lld milliseconds. You can try killing the script using the SCRIPT KILL command.",elapsed); server.lua_timedout = 1; /* Once the script timeouts we reenter the event loop to permit others @@ -1351,9 +1346,7 @@ void evalGenericCommand(client *c, int evalsha) { server.lua_caller = c; server.lua_time_start = mstime(); server.lua_kill = 0; - if (server.lua_time_limit > 0 && server.masterhost == NULL && - ldb.active == 0) - { + if (server.lua_time_limit > 0 && ldb.active == 0) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); delhook = 1; } else if (ldb.active) { @@ -1503,6 +1496,8 @@ NULL } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { if (server.lua_caller == NULL) { addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n")); + } else if (server.lua_caller->flags & CLIENT_MASTER) { + addReplySds(c,sdsnew("-UNKILLABLE The busy script was sent by a master instance in the context of replication and cannot be killed.\r\n")); } else if (server.lua_write_dirty) { addReplySds(c,sdsnew("-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command.\r\n")); } else { From e789b562f38077879ef06f63bb210b7a4576dc0b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 Aug 2018 17:40:58 +0200 Subject: [PATCH 105/616] While the slave is busy, just accumulate master input. Processing command from the master while the slave is in busy state is not correct, however we cannot, also, just reply -BUSY to the replication stream commands from the master. The correct solution is to stop processing data from the master, but just accumulate the stream into the buffers and resume the processing later. Related to #5297. --- src/networking.c | 6 ++++++ src/server.c | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 58248ced0..eb581462b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1362,6 +1362,12 @@ void processInputBuffer(client *c) { /* Immediately abort if the client is in the middle of something. */ if (c->flags & CLIENT_BLOCKED) break; + /* Don't process input from the master while there is a busy script + * condition on the slave. We want just to accumulate the replication + * stream (instead of replying -BUSY like we do with other clients) and + * later resume the processing. */ + if (server.lua_timedout && c->flags & CLIENT_MASTER) break; + /* CLIENT_CLOSE_AFTER_REPLY closes the connection once the reply is * written to the client. Make sure to not let the reply grow after * this flag has been set (i.e. don't process more commands). diff --git a/src/server.c b/src/server.c index 1afc9a802..55bcb4dc2 100644 --- a/src/server.c +++ b/src/server.c @@ -2683,7 +2683,6 @@ int processCommand(client *c) { /* Lua script too slow? Only allow a limited number of commands. */ if (server.lua_timedout && - !(c->flags & CLIENT_MASTER) && c->cmd->proc != authCommand && c->cmd->proc != replconfCommand && !(c->cmd->proc == shutdownCommand && From 99316560f199791d064e027bedb3dfb4aa217acf Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 Aug 2018 16:07:03 +0200 Subject: [PATCH 106/616] After slave Lua script leaves busy state, re-process the master buffer. Technically speaking we don't really need to put the master client in the clients that need to be processed, since in practice the PING commands from the master will take care, however it is conceptually more sane to do so. --- src/networking.c | 3 +-- src/scripting.c | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index eb581462b..27c695306 100644 --- a/src/networking.c +++ b/src/networking.c @@ -827,8 +827,7 @@ void freeClient(client *c) { serverLog(LL_WARNING,"Connection with master lost."); if (!(c->flags & (CLIENT_CLOSE_AFTER_REPLY| CLIENT_CLOSE_ASAP| - CLIENT_BLOCKED| - CLIENT_UNBLOCKED))) + CLIENT_BLOCKED))) { replicationCacheMaster(c); return; diff --git a/src/scripting.c b/src/scripting.c index 0ef8d2a6f..4b36a085b 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,6 +1367,10 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); + if (server.masterhost && server.master) { + server.master->flags |= CLIENT_UNBLOCKED; + listAddNodeTail(server.unblocked_clients,server.master); + } } server.lua_caller = NULL; From 3880af78087b3f1246e5747d8c9325be7fc0cc24 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 Aug 2018 16:43:38 +0200 Subject: [PATCH 107/616] Test: processing of master stream in slave -BUSY state. See #5297. --- tests/integration/replication.tcl | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index e811cf0ee..56d2de1cc 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -266,3 +266,47 @@ foreach dl {no yes} { } } } + +start_server {tags {"repl"}} { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set load_handle0 [start_write_load $master_host $master_port 3] + start_server {} { + test "Master stream is correctly processed while the slave has a script in -BUSY state" { + set slave [srv 0 client] + puts [srv 0 port] + $slave config set lua-time-limit 500 + $slave slaveof $master_host $master_port + + # Wait for the slave to be online + wait_for_condition 500 100 { + [lindex [$slave role] 3] eq {connected} + } else { + fail "Slave still not connected after some time" + } + + # Wait some time to make sure the master is sending data + # to the slave. + after 5000 + + # Stop the ability of the slave to process data by sendig + # a script that will put it in BUSY state. + $slave eval {for i=1,3000000000 do end} 0 + + # Wait some time again so that more master stream will + # be processed. + after 2000 + + # Stop the write load + stop_write_load $load_handle0 + + # number of keys + wait_for_condition 500 100 { + [$master debug digest] eq [$slave debug digest] + } else { + fail "Different datasets between slave and master" + } + } + } +} From 410e2a9cbbc74666f55ee87fb9a68c4ee2178a42 Mon Sep 17 00:00:00 2001 From: Amin Mesbah Date: Sun, 2 Sep 2018 00:06:20 -0700 Subject: [PATCH 108/616] Use geohash limit defines in constraint check Slight robustness improvement, especially if the limit values are changed, as was suggested in antires/redis#4291 [1]. [1] https://github.com/antirez/redis/pull/4291 --- src/geohash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geohash.c b/src/geohash.c index b40282e76..db5ae025a 100644 --- a/src/geohash.c +++ b/src/geohash.c @@ -127,8 +127,8 @@ int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range, /* Return an error when trying to index outside the supported * constraints. */ - if (longitude > 180 || longitude < -180 || - latitude > 85.05112878 || latitude < -85.05112878) return 0; + if (longitude > GEO_LONG_MAX || longitude < GEO_LONG_MIN || + latitude > GEO_LAT_MAX || latitude < GEO_LAT_MIN) return 0; hash->bits = 0; hash->step = step; From 145a5e01e36fd7ce730918757ac7e873a89d05ca Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 6 Nov 2017 16:21:35 +0800 Subject: [PATCH 109/616] fix multiple unblock for clientsArePaused() --- src/networking.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 27c695306..36228d6f0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2126,9 +2126,9 @@ int clientsArePaused(void) { while ((ln = listNext(&li)) != NULL) { c = listNodeValue(ln); - /* Don't touch slaves and blocked clients. The latter pending - * requests be processed when unblocked. */ - if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue; + /* Don't touch slaves and blocked or unblocked clients. + * The latter pending requests be processed when unblocked. */ + if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED|CLIENT_UNBLOCKED)) continue; c->flags |= CLIENT_UNBLOCKED; listAddNodeTail(server.unblocked_clients,c); } From 2a40cc26c53cd08966b7eda60eaa65619c6fe67c Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Mon, 3 Sep 2018 14:36:48 +0800 Subject: [PATCH 110/616] if master is already unblocked, do not unblock it twice --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 4b36a085b..6c311dbe0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,7 +1367,7 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); - if (server.masterhost && server.master) { + if (server.masterhost && server.master && !(server.master->flags & CLIENT_UNBLOCKED)) { server.master->flags |= CLIENT_UNBLOCKED; listAddNodeTail(server.unblocked_clients,server.master); } From a26756ab59e6b0c2cc501956308775d614b1bcc7 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Tue, 4 Sep 2018 00:02:18 +0800 Subject: [PATCH 111/616] networking: optimize parsing large bulk greater than 32k If we are going to read a large object from network try to make it likely that it will start at c->querybuf boundary so that we can optimize object creation avoiding a large copy of data. But only when the data we have not parsed is less than or equal to ll+2. If the data length is greater than ll+2, trimming querybuf is just a waste of time, because at this time the querybuf contains not only our bulk. It's easy to reproduce the that: Time1: call `client pause 10000` on slave. Time2: redis-benchmark -t set -r 10000 -d 33000 -n 10000. Then slave hung after 10 seconds. --- src/networking.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/networking.c b/src/networking.c index 27c695306..eac94798c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1294,19 +1294,22 @@ int processMultibulkBuffer(client *c) { c->qb_pos = newline-c->querybuf+2; if (ll >= PROTO_MBULK_BIG_ARG) { - size_t qblen; - /* If we are going to read a large object from network * try to make it likely that it will start at c->querybuf * boundary so that we can optimize object creation - * avoiding a large copy of data. */ - sdsrange(c->querybuf,c->qb_pos,-1); - c->qb_pos = 0; - qblen = sdslen(c->querybuf); - /* Hint the sds library about the amount of bytes this string is - * going to contain. */ - if (qblen < (size_t)ll+2) - c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-qblen); + * avoiding a large copy of data. + * + * But only when the data we have not parsed is less than + * or equal to ll+2. If the data length is greater than + * ll+2, trimming querybuf is just a waste of time, because + * at this time the querybuf contains not only our bulk. */ + if (sdslen(c->querybuf)-c->qb_pos <= (size_t)ll+2) { + sdsrange(c->querybuf,c->qb_pos,-1); + c->qb_pos = 0; + /* Hint the sds library about the amount of bytes this string is + * going to contain. */ + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2); + } } c->bulklen = ll; } From f0348a65436da6b608c44c67bd09f810cd31d740 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Sep 2018 18:17:25 +0200 Subject: [PATCH 112/616] Make pending buffer processing safe for CLIENT_MASTER client. Related to #5305. --- src/blocked.c | 2 +- src/networking.c | 32 ++++++++++++++++++++------------ src/server.h | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index 35c33d1cc..aeca87a6e 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -126,7 +126,7 @@ void processUnblockedClients(void) { * the code is conceptually more correct this way. */ if (!(c->flags & CLIENT_BLOCKED)) { if (c->querybuf && sdslen(c->querybuf) > 0) { - processInputBuffer(c); + processInputBufferAndReplicate(c); } } } diff --git a/src/networking.c b/src/networking.c index 27c695306..8e55ec902 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1425,6 +1425,25 @@ void processInputBuffer(client *c) { server.current_client = NULL; } +/* This is a wrapper for processInputBuffer that also cares about handling + * the replication forwarding to the sub-slaves, in case the client 'c' + * is flagged as master. Usually you want to call this instead of the + * raw processInputBuffer(). */ +void processInputBufferAndReplicate(client *c) { + if (!(c->flags & CLIENT_MASTER)) { + processInputBuffer(c); + } else { + size_t prev_offset = c->reploff; + processInputBuffer(c); + size_t applied = c->reploff - prev_offset; + if (applied) { + replicationFeedSlavesFromMasterStream(server.slaves, + c->pending_querybuf, applied); + sdsrange(c->pending_querybuf,applied,-1); + } + } +} + void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { client *c = (client*) privdata; int nread, readlen; @@ -1492,18 +1511,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { * was actually applied to the master state: this quantity, and its * corresponding part of the replication stream, will be propagated to * the sub-slaves and to the replication backlog. */ - if (!(c->flags & CLIENT_MASTER)) { - processInputBuffer(c); - } else { - size_t prev_offset = c->reploff; - processInputBuffer(c); - size_t applied = c->reploff - prev_offset; - if (applied) { - replicationFeedSlavesFromMasterStream(server.slaves, - c->pending_querybuf, applied); - sdsrange(c->pending_querybuf,applied,-1); - } - } + processInputBufferAndReplicate(c); } void getClientsMaxBuffers(unsigned long *longest_output_list, diff --git a/src/server.h b/src/server.h index 4fb8e383c..4a5967f10 100644 --- a/src/server.h +++ b/src/server.h @@ -1418,6 +1418,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask); void *addDeferredMultiBulkLength(client *c); void setDeferredMultiBulkLength(client *c, void *node, long length); void processInputBuffer(client *c); +void processInputBufferAndReplicate(client *c); void acceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask); void acceptUnixHandler(aeEventLoop *el, int fd, void *privdata, int mask); From 02306736c96e2d259a09e2e095dd7219fe57150e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Sep 2018 18:39:18 +0200 Subject: [PATCH 113/616] Unblocked clients API refactoring. See #4418. --- src/blocked.c | 32 ++++++++++++++++++++++++++------ src/networking.c | 9 ++++----- src/scripting.c | 6 ++---- src/server.h | 1 + 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index aeca87a6e..00212ed69 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -132,6 +132,31 @@ void processUnblockedClients(void) { } } +/* This function will schedule the client for reprocessing at a safe time. + * + * This is useful when a client was blocked for some reason (blocking opeation, + * CLIENT PAUSE, or whatever), because it may end with some accumulated query + * buffer that needs to be processed ASAP: + * + * 1. When a client is blocked, its readable handler is still active. + * 2. However in this case it only gets data into the query buffer, but the + * query is not parsed or executed once there is enough to proceed as + * usually (because the client is blocked... so we can't execute commands). + * 3. When the client is unblocked, without this function, the client would + * have to write some query in order for the readable handler to finally + * call processQueryBuffer*() on it. + * 4. With this function instead we can put the client in a queue that will + * process it for queries ready to be executed at a safe time. + */ +void queueClientForReprocessing(client *c) { + /* The client may already be into the unblocked list because of a previous + * blocking operation, don't add back it into the list multiple times. */ + if (!(c->flags & CLIENT_UNBLOCKED)) { + c->flags |= CLIENT_UNBLOCKED; + listAddNodeTail(server.unblocked_clients,c); + } +} + /* Unblock a client calling the right function depending on the kind * of operation the client is blocking for. */ void unblockClient(client *c) { @@ -152,12 +177,7 @@ void unblockClient(client *c) { server.blocked_clients_by_type[c->btype]--; c->flags &= ~CLIENT_BLOCKED; c->btype = BLOCKED_NONE; - /* The client may already be into the unblocked list because of a previous - * blocking operation, don't add back it into the list multiple times. */ - if (!(c->flags & CLIENT_UNBLOCKED)) { - c->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,c); - } + queueClientForReprocessing(c); } /* This function gets called when a blocked client timed out in order to diff --git a/src/networking.c b/src/networking.c index 824d88245..0c1b3016f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2134,11 +2134,10 @@ int clientsArePaused(void) { while ((ln = listNext(&li)) != NULL) { c = listNodeValue(ln); - /* Don't touch slaves and blocked or unblocked clients. - * The latter pending requests be processed when unblocked. */ - if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED|CLIENT_UNBLOCKED)) continue; - c->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,c); + /* Don't touch slaves and blocked clients. + * The latter pending requests will be processed when unblocked. */ + if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue; + queueClientForReprocessing(c); } } return server.clients_paused; diff --git a/src/scripting.c b/src/scripting.c index 6c311dbe0..8aa62071c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1367,10 +1367,8 @@ void evalGenericCommand(client *c, int evalsha) { * script timeout was detected. */ aeCreateFileEvent(server.el,c->fd,AE_READABLE, readQueryFromClient,c); - if (server.masterhost && server.master && !(server.master->flags & CLIENT_UNBLOCKED)) { - server.master->flags |= CLIENT_UNBLOCKED; - listAddNodeTail(server.unblocked_clients,server.master); - } + if (server.masterhost && server.master) + queueClientForReprocessing(server.master); } server.lua_caller = NULL; diff --git a/src/server.h b/src/server.h index 4a5967f10..09348585b 100644 --- a/src/server.h +++ b/src/server.h @@ -1884,6 +1884,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body); void processUnblockedClients(void); void blockClient(client *c, int btype); void unblockClient(client *c); +void queueClientForReprocessing(client *c); void replyToBlockedClientTimedOut(client *c); int getTimeoutFromObjectOrReply(client *c, robj *object, mstime_t *timeout, int unit); void disconnectAllBlockedClients(void); From 90a82823d6a5ffc16001cf6aaa69d9d247ff6f5d Mon Sep 17 00:00:00 2001 From: maya-rv <42264341+maya-rv@users.noreply.github.com> Date: Tue, 4 Sep 2018 13:32:02 +0300 Subject: [PATCH 114/616] Fix typo --- tests/support/server.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 5578f1fd6..0edb25d8a 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -4,7 +4,7 @@ set ::valgrind_errors {} proc start_server_error {config_file error} { set err {} - append err "Cant' start the Redis server\n" + append err "Can't start the Redis server\n" append err "CONFIGURATION:" append err [exec cat $config_file] append err "\nERROR:" From cbbf49b7d3910e01b0f7aefd9654afd6780895da Mon Sep 17 00:00:00 2001 From: Sascha Roland Date: Wed, 29 Aug 2018 17:53:47 +0200 Subject: [PATCH 115/616] #5299 Fix blocking XREAD for streams that ran dry The conclusion, that a xread request can be answered syncronously in case that the stream's last_id is larger than the passed last-received-id parameter, assumes, that there must be entries present, which could be returned immediately. This assumption fails for empty streams that actually contained some entries which got removed by xdel, ... . As result, the client is answered synchronously with an empty result, instead of blocking for new entries to arrive. An additional check for a non-empty stream is required. --- src/t_stream.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 77fbf4645..40e0c84b8 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -1436,7 +1436,7 @@ void xreadCommand(client *c) { * synchronously in case the group top item delivered is smaller * than what the stream has inside. */ streamID *last = &groups[i]->last_id; - if (streamCompareID(&s->last_id, last) > 0) { + if (s->length && (streamCompareID(&s->last_id, last) > 0)) { serve_synchronously = 1; *gt = *last; } @@ -1444,7 +1444,7 @@ void xreadCommand(client *c) { } else { /* For consumers without a group, we serve synchronously if we can * actually provide at least one item from the stream. */ - if (streamCompareID(&s->last_id, gt) > 0) { + if (s->length && (streamCompareID(&s->last_id, gt) > 0)) { serve_synchronously = 1; } } From 5fda4a85dd8e6902e8f47b788ff33b4e5904edf9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Sep 2018 13:29:24 +0200 Subject: [PATCH 116/616] Clarify why remaining may be zero in readQueryFromClient(). See #5304. --- src/networking.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/networking.c b/src/networking.c index 605621c1c..01307c7a3 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1466,6 +1466,8 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { { ssize_t remaining = (size_t)(c->bulklen+2)-sdslen(c->querybuf); + /* Note that the 'remaining' variable may be zero in some edge case, + * for example once we resume a blocked client after CLIENT PAUSE. */ if (remaining > 0 && remaining < readlen) readlen = remaining; } From 219c5fbb1b5ab249bdc69f365735c6deb1d2b237 Mon Sep 17 00:00:00 2001 From: youjiali1995 Date: Wed, 5 Sep 2018 10:32:18 +0800 Subject: [PATCH 117/616] sentinel: fix randomized sentinelTimer. --- src/server.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/server.c b/src/server.c index 55bcb4dc2..cff218446 100644 --- a/src/server.c +++ b/src/server.c @@ -1320,9 +1320,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } /* Run the Sentinel timer if we are in sentinel mode. */ - run_with_period(100) { - if (server.sentinel_mode) sentinelTimer(); - } + if (server.sentinel_mode) sentinelTimer(); /* Cleanup expired MIGRATE cached sockets. */ run_with_period(1000) { From ac7b3ec9ca9488801fb19ba371b1ebec7c01feab Mon Sep 17 00:00:00 2001 From: youjiali1995 Date: Wed, 5 Sep 2018 16:51:13 +0800 Subject: [PATCH 118/616] bio: fix bioWaitStepOfType. --- src/bio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bio.c b/src/bio.c index 0c92d053b..0e15c416d 100644 --- a/src/bio.c +++ b/src/bio.c @@ -204,14 +204,14 @@ void *bioProcessBackgroundJobs(void *arg) { } zfree(job); - /* Unblock threads blocked on bioWaitStepOfType() if any. */ - pthread_cond_broadcast(&bio_step_cond[type]); - /* Lock again before reiterating the loop, if there are no longer * jobs to process we'll block again in pthread_cond_wait(). */ pthread_mutex_lock(&bio_mutex[type]); listDelNode(bio_jobs[type],ln); bio_pending[type]--; + + /* Unblock threads blocked on bioWaitStepOfType() if any. */ + pthread_cond_broadcast(&bio_step_cond[type]); } } From 042ee28b09cac5591f227b3bf95f2eaa2afebc24 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 13:10:05 +0200 Subject: [PATCH 119/616] Don't perform eviction when re-entering the event loop. Related to #5250. --- src/server.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/server.c b/src/server.c index 55bcb4dc2..fd8472f60 100644 --- a/src/server.c +++ b/src/server.c @@ -2593,8 +2593,13 @@ int processCommand(client *c) { * * First we try to free some memory if possible (if there are volatile * keys in the dataset). If there are not the only thing we can do - * is returning an error. */ - if (server.maxmemory) { + * is returning an error. + * + * Note that we do not want to reclaim memory if we are here re-entering + * the event loop since there is a busy Lua script running in timeout + * condition, to avoid mixing the propagation of scripts with the propagation + * of DELs due to eviction. */ + if (server.maxmemory && !server.lua_timedout) { int out_of_memory = freeMemoryIfNeeded() == C_ERR; /* freeMemoryIfNeeded may flush slave output buffers. This may result * into a slave, that may be the active client, to be freed. */ From a8a0b52358718198149cb42e9b8de13d0961ad4e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 13:20:12 +0200 Subject: [PATCH 120/616] Propagate read-only scripts as SCRIPT LOAD. See issue #5250 and the new comments added to the code in this commit for details. --- src/scripting.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 8aa62071c..4b5b0002f 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1250,6 +1250,7 @@ void evalGenericCommand(client *c, int evalsha) { lua_State *lua = server.lua; char funcname[43]; long long numkeys; + long long initial_server_dirty = server.dirty; int delhook = 0, err; /* When we replicate whole scripts, we want the same PRNG sequence at @@ -1432,9 +1433,21 @@ void evalGenericCommand(client *c, int evalsha) { replicationScriptCacheAdd(c->argv[1]->ptr); serverAssertWithInfo(c,NULL,script != NULL); - rewriteClientCommandArgument(c,0, - resetRefCount(createStringObject("EVAL",4))); - rewriteClientCommandArgument(c,1,script); + + /* If the script did not produce any changes in the dataset we want + * just to replicate it as SCRIPT LOAD, otherwise we risk running + * an aborted script on slaves (that may then produce results there) + * or just running a CPU costly read-only script on the slaves. */ + if (server.dirty == initial_server_dirty) { + rewriteClientCommandVector(c,3, + resetRefCount(createStringObject("SCRIPT",6)), + resetRefCount(createStringObject("LOAD",4)), + script); + } else { + rewriteClientCommandArgument(c,0, + resetRefCount(createStringObject("EVAL",4))); + rewriteClientCommandArgument(c,1,script); + } forceCommandPropagation(c,PROPAGATE_REPL|PROPAGATE_AOF); } } From 1fd6c390b29d75d096dcaa8b811728c01a681f9b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 15:48:08 +0200 Subject: [PATCH 121/616] Safer script stop condition on OOM. Here the idea is that we do not want freeMemoryIfNeeded() to propagate a DEL command before the script and change what happens in the script execution once it reaches the slave. For example see this potential issue (in the words of @soloestoy): On master, we run the following script: if redis.call('get','key') then redis.call('set','xxx','yyy') end redis.call('set','c','d') Then when redis attempts to execute redis.call('set','xxx','yyy'), we call freeMemoryIfNeeded(), and the key may get deleted, and because redis.call('set','xxx','yyy') has already been executed on master, this script will be replicated to slave. But the slave received "DEL key" before the script, and will ignore maxmemory, so after that master has xxx and c, slave has only one key c. Note that this patch (and other related work) was authored collaboratively in issue #5250 with the help of @soloestoy and @oranagra. Related to issue #5250. --- src/scripting.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 4b5b0002f..43f8d09a5 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -512,10 +512,13 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { * could enlarge the memory usage are not allowed, but only if this is the * first write in the context of this script, otherwise we can't stop * in the middle. */ - if (server.maxmemory && server.lua_write_dirty == 0 && + if (server.maxmemory && /* Maxmemory is actually enabled. */ + !server.loading && /* Don't care about mem if loading. */ + !server.masterhost && /* Slave must execute the script. */ + server.lua_write_dirty == 0 && /* Script had no side effects so far. */ (cmd->flags & CMD_DENYOOM)) { - if (freeMemoryIfNeeded() == C_ERR) { + if (getMaxmemoryState(NULL,NULL,NULL,NULL) != C_OK) { luaPushError(lua, shared.oomerr->ptr); goto cleanup; } From 39e09ea84c21fed1244d10778b52084ff344f17a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 19:33:56 +0200 Subject: [PATCH 122/616] Use commands (effects) replication by default in scripts. See issue #5250 and issue #5292 for more info. --- src/config.c | 2 ++ src/scripting.c | 1 - src/server.c | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 1a64d25ed..b4c5b404a 100644 --- a/src/config.c +++ b/src/config.c @@ -677,6 +677,8 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"lua-replicate-commands") && argc == 2) { + server.lua_always_replicate_commands = yesnotoi(argv[1]); } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { diff --git a/src/scripting.c b/src/scripting.c index 43f8d09a5..acea551ea 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -917,7 +917,6 @@ void scriptingInit(int setup) { server.lua_client = NULL; server.lua_caller = NULL; server.lua_timedout = 0; - server.lua_always_replicate_commands = 0; /* Only DEBUG can change it.*/ ldbInit(); } diff --git a/src/server.c b/src/server.c index fd8472f60..af5bef7f9 100644 --- a/src/server.c +++ b/src/server.c @@ -1715,6 +1715,12 @@ void initServerConfig(void) { server.assert_line = 0; server.bug_report_start = 0; server.watchdog_period = 0; + + /* By default we want scripts to be always replicated by effects + * (single commands executed by the script), and not by sending the + * script to the slave / AOF. This is the new way starting from + * Redis 5. However it is possible to revert it via redis.conf. */ + server.lua_always_replicate_commands = 1; } extern char **environ; From d5b60e1a4db118d165a3866124fa9bf048964910 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 5 Sep 2018 19:47:29 +0200 Subject: [PATCH 123/616] Fix scripting tests now that we default to commands repl. --- tests/unit/scripting.tcl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index bcde721c3..b0591adfa 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -148,9 +148,11 @@ start_server {tags {"scripting"}} { test {EVAL - Scripts can't run certain commands} { set e {} + r debug lua-always-replicate-commands 0 catch { r eval "redis.pcall('randomkey'); return redis.pcall('set','x','ciao')" 0 } e + r debug lua-always-replicate-commands 1 set e } {*not allowed after*} @@ -299,9 +301,12 @@ start_server {tags {"scripting"}} { } {b534286061d4b9e4026607613b95c06c06015ae8 loaded} test "In the context of Lua the output of random commands gets ordered" { + r debug lua-always-replicate-commands 0 r del myset r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz - r eval {return redis.call('smembers',KEYS[1])} 1 myset + set res [r eval {return redis.call('smembers',KEYS[1])} 1 myset] + r debug lua-always-replicate-commands 1 + set res } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} test "SORT is normally not alpha re-ordered for the scripting engine" { @@ -655,11 +660,13 @@ start_server {tags {"scripting repl"}} { } {1} test "Redis.set_repl() must be issued after replicate_commands()" { + r debug lua-always-replicate-commands 0 catch { r eval { redis.set_repl(redis.REPL_ALL); } 0 } e + r debug lua-always-replicate-commands 1 set e } {*only after turning on*} From 82634bcc19a6b367fbb7c189f8ab634ed5eb6940 Mon Sep 17 00:00:00 2001 From: Weiliang Li Date: Thu, 6 Sep 2018 13:40:05 +0800 Subject: [PATCH 124/616] fix usage typo in redis-cli --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0e8777bd2..2bb8c56f8 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1495,7 +1495,7 @@ static void usage(void) { " --ldb Used with --eval enable the Redis Lua debugger.\n" " --ldb-sync-mode Like --ldb but uses the synchronous Lua debugger, in\n" " this mode the server is blocked and script changes are\n" -" are not rolled back from the server memory.\n" +" not rolled back from the server memory.\n" " --cluster [args...] [opts...]\n" " Cluster Manager command and arguments (see below).\n" " --verbose Verbose mode.\n" From f3a78379820ecb190a240664a0d32683885f82c8 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" Date: Thu, 6 Sep 2018 20:52:05 +0800 Subject: [PATCH 125/616] Scripting & Streams: some commands need right flags xadd with id * generates random stream id xadd & xtrim with approximate maxlen count may trim stream randomly xinfo may get random radix-tree-keys/nodes xpending may get random idletime xclaim: master and slave may have different idletime in stream --- src/server.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server.c b/src/server.c index af5bef7f9..b65d69a16 100644 --- a/src/server.c +++ b/src/server.c @@ -306,7 +306,7 @@ struct redisCommand redisCommandTable[] = { {"pfcount",pfcountCommand,-2,"r",0,NULL,1,-1,1,0,0}, {"pfmerge",pfmergeCommand,-2,"wm",0,NULL,1,-1,1,0,0}, {"pfdebug",pfdebugCommand,-3,"w",0,NULL,0,0,0,0,0}, - {"xadd",xaddCommand,-5,"wmF",0,NULL,1,1,1,0,0}, + {"xadd",xaddCommand,-5,"wmFR",0,NULL,1,1,1,0,0}, {"xrange",xrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xrevrange",xrevrangeCommand,-4,"r",0,NULL,1,1,1,0,0}, {"xlen",xlenCommand,2,"rF",0,NULL,1,1,1,0,0}, @@ -314,11 +314,11 @@ struct redisCommand redisCommandTable[] = { {"xreadgroup",xreadCommand,-7,"ws",0,xreadGetKeys,1,1,1,0,0}, {"xgroup",xgroupCommand,-2,"wm",0,NULL,2,2,1,0,0}, {"xack",xackCommand,-4,"wF",0,NULL,1,1,1,0,0}, - {"xpending",xpendingCommand,-3,"r",0,NULL,1,1,1,0,0}, - {"xclaim",xclaimCommand,-6,"wF",0,NULL,1,1,1,0,0}, - {"xinfo",xinfoCommand,-2,"r",0,NULL,2,2,1,0,0}, + {"xpending",xpendingCommand,-3,"rR",0,NULL,1,1,1,0,0}, + {"xclaim",xclaimCommand,-6,"wRF",0,NULL,1,1,1,0,0}, + {"xinfo",xinfoCommand,-2,"rR",0,NULL,2,2,1,0,0}, {"xdel",xdelCommand,-3,"wF",0,NULL,1,1,1,0,0}, - {"xtrim",xtrimCommand,-2,"wF",0,NULL,1,1,1,0,0}, + {"xtrim",xtrimCommand,-2,"wFR",0,NULL,1,1,1,0,0}, {"post",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"host:",securityWarningCommand,-1,"lt",0,NULL,0,0,0,0,0}, {"latency",latencyCommand,-2,"aslt",0,NULL,0,0,0,0,0} From bdca5b9be5886a2d1e4e0e166f4fc8dc634a6ca9 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:43:39 +0200 Subject: [PATCH 126/616] Slave removal: SLAVEOF -> REPLICAOF. SLAVEOF is now an alias. --- src/replication.c | 2 +- src/server.c | 3 ++- src/server.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/replication.c b/src/replication.c index 6d589c012..11fd34791 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1997,7 +1997,7 @@ void replicationHandleMasterDisconnection(void) { * the slaves only if we'll have to do a full resync with our master. */ } -void slaveofCommand(client *c) { +void replicaofCommand(client *c) { /* SLAVEOF is not allowed in cluster mode as replication is automatically * configured using the current address of the master node. */ if (server.cluster_enabled) { diff --git a/src/server.c b/src/server.c index 9d2a3718d..92a3b17c0 100644 --- a/src/server.c +++ b/src/server.c @@ -260,7 +260,8 @@ struct redisCommand redisCommandTable[] = { {"touch",touchCommand,-2,"rF",0,NULL,1,1,1,0,0}, {"pttl",pttlCommand,2,"rFR",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"wF",0,NULL,1,1,1,0,0}, - {"slaveof",slaveofCommand,3,"ast",0,NULL,0,0,0,0,0}, + {"slaveof",replicaofCommand,3,"ast",0,NULL,0,0,0,0,0}, + {"replicaof",replicaofCommand,3,"ast",0,NULL,0,0,0,0,0}, {"role",roleCommand,1,"lst",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"last",0,NULL,0,0,0,0,0}, diff --git a/src/server.h b/src/server.h index 09348585b..ae20aff6a 100644 --- a/src/server.h +++ b/src/server.h @@ -1998,7 +1998,7 @@ void ttlCommand(client *c); void touchCommand(client *c); void pttlCommand(client *c); void persistCommand(client *c); -void slaveofCommand(client *c); +void replicaofCommand(client *c); void roleCommand(client *c); void debugCommand(client *c); void msetCommand(client *c); From d0647506b35d5835d1bc4a79da203a6abda63d7c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:46:28 +0200 Subject: [PATCH 127/616] Slave removal: redis-cli --slave -> --replica. --slave alias remains but is undocumented, just for backward compatibiltiy. --- src/redis-cli.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 536eeb79f..0a3d78db3 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1282,6 +1282,8 @@ static int parseOptions(int argc, char **argv) { config.lru_test_sample_size = strtoll(argv[++i],NULL,10); } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--replica")) { + config.slave_mode = 1; } else if (!strcmp(argv[i],"--stat")) { config.stat_mode = 1; } else if (!strcmp(argv[i],"--scan")) { @@ -1478,7 +1480,7 @@ static void usage(void) { " --latency-dist Shows latency as a spectrum, requires xterm 256 colors.\n" " Default time interval is 1 sec. Change it using -i.\n" " --lru-test Simulate a cache workload with an 80-20 distribution.\n" -" --slave Simulate a slave showing commands received from the master.\n" +" --replica Simulate a replica showing commands received from the master.\n" " --rdb Transfer an RDB dump from remote server to local file.\n" " --pipe Transfer raw Redis protocol from stdin to server.\n" " --pipe-timeout In --pipe mode, abort with error if after sending all data.\n" From 3b87cad43d76fcfb7688274a96e9063a888d6de2 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 10:49:03 +0200 Subject: [PATCH 128/616] Slave removal: Convert cluster.c log messages and command names. --- src/cluster.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cluster.c b/src/cluster.c index 2f3e298e0..421e678bb 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -1230,7 +1230,7 @@ void clearNodeFailureIfNeeded(clusterNode *node) { serverLog(LL_NOTICE, "Clear FAIL state for node %.40s: %s is reachable again.", node->name, - nodeIsSlave(node) ? "slave" : "master without slots"); + nodeIsSlave(node) ? "replica" : "master without slots"); node->flags &= ~CLUSTER_NODE_FAIL; clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); } @@ -2059,7 +2059,7 @@ int clusterProcessPacket(clusterLink *link) { server.cluster->mf_end = mstime() + CLUSTER_MF_TIMEOUT; server.cluster->mf_slave = sender; pauseClients(mstime()+(CLUSTER_MF_TIMEOUT*2)); - serverLog(LL_WARNING,"Manual failover requested by slave %.40s.", + serverLog(LL_WARNING,"Manual failover requested by replica %.40s.", sender->name); } else if (type == CLUSTERMSG_TYPE_UPDATE) { clusterNode *n; /* The node the update is about. */ @@ -2873,7 +2873,7 @@ void clusterLogCantFailover(int reason) { switch(reason) { case CLUSTER_CANT_FAILOVER_DATA_AGE: msg = "Disconnected from master for longer than allowed. " - "Please check the 'cluster-slave-validity-factor' configuration " + "Please check the 'cluster-replica-validity-factor' configuration " "option."; break; case CLUSTER_CANT_FAILOVER_WAITING_DELAY: @@ -3054,7 +3054,7 @@ void clusterHandleSlaveFailover(void) { server.cluster->failover_auth_time += added_delay; server.cluster->failover_auth_rank = newrank; serverLog(LL_WARNING, - "Slave rank updated to #%d, added %lld milliseconds of delay.", + "Replica rank updated to #%d, added %lld milliseconds of delay.", newrank, added_delay); } } @@ -4187,7 +4187,7 @@ void clusterCommand(client *c) { "COUNT-failure-reports -- Return number of failure reports for .", "COUNTKEYSINSLOT - Return the number of keys in .", "DELSLOTS [slot ...] -- Delete slots information from current node.", -"FAILOVER [force|takeover] -- Promote current slave node to being a master.", +"FAILOVER [force|takeover] -- Promote current replica node to being a master.", "FORGET -- Remove a node from the cluster.", "GETKEYSINSLOT -- Return key names stored by current node in a slot.", "FLUSHSLOTS -- Delete current node own slots information.", @@ -4197,11 +4197,11 @@ void clusterCommand(client *c) { "MYID -- Return the node id.", "NODES -- Return cluster configuration seen by node. Output format:", " ... ", -"REPLICATE -- Configure current node as slave to .", +"REPLICATE -- Configure current node as replica to .", "RESET [hard|soft] -- Reset current node (default: soft).", "SET-config-epoch - Set config epoch of current node.", "SETSLOT (importing|migrating|stable|node ) -- Set slot state.", -"SLAVES -- Return slaves.", +"REPLICAS -- Return replicas.", "SLOTS -- Return information about slots range mappings. Each range is made of:", " start, end, master and replicas IP addresses, ports and ids", NULL @@ -4578,7 +4578,7 @@ NULL /* Can't replicate a slave. */ if (nodeIsSlave(n)) { - addReplyError(c,"I can only replicate a master, not a slave."); + addReplyError(c,"I can only replicate a master, not a replica."); return; } @@ -4597,7 +4597,8 @@ NULL clusterSetMaster(n); clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"slaves") && c->argc == 3) { + } else if ((!strcasecmp(c->argv[1]->ptr,"slaves") || + !strcasecmp(c->argv[1]->ptr,"replicas")) && c->argc == 3) { /* CLUSTER SLAVES */ clusterNode *n = clusterLookupNode(c->argv[2]->ptr); int j; @@ -4651,10 +4652,10 @@ NULL /* Check preconditions. */ if (nodeIsMaster(myself)) { - addReplyError(c,"You should send CLUSTER FAILOVER to a slave"); + addReplyError(c,"You should send CLUSTER FAILOVER to a replica"); return; } else if (myself->slaveof == NULL) { - addReplyError(c,"I'm a slave but my master is unknown to me"); + addReplyError(c,"I'm a replica but my master is unknown to me"); return; } else if (!force && (nodeFailed(myself->slaveof) || From 33cfd1a30eaa52b56b40f5c88fc1c8621bf89087 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:22:20 +0200 Subject: [PATCH 129/616] Slave removal: config.c converted + config rewriting hacks. Aliases added for all the commands mentioning slave. Moreover CONFIG REWRITE will use the new names, and will be able to reuse the old lines mentioning the old options. --- src/config.c | 155 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 117 insertions(+), 38 deletions(-) diff --git a/src/config.c b/src/config.c index b4c5b404a..3b38f98d8 100644 --- a/src/config.c +++ b/src/config.c @@ -344,15 +344,19 @@ void loadServerConfigFromString(char *config) { err = "lfu-decay-time must be 0 or greater"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slaveof") && argc == 3) { + } else if ((!strcasecmp(argv[0],"slaveof") || + !strcasecmp(argv[0],"replicaof")) && argc == 3) { slaveof_linenum = linenum; server.masterhost = sdsnew(argv[1]); server.masterport = atoi(argv[2]); server.repl_state = REPL_STATE_CONNECT; - } else if (!strcasecmp(argv[0],"repl-ping-slave-period") && argc == 2) { + } else if ((!strcasecmp(argv[0],"repl-ping-slave-period") || + !strcasecmp(argv[0],"repl-ping-replica-period")) && + argc == 2) + { server.repl_ping_slave_period = atoi(argv[1]); if (server.repl_ping_slave_period <= 0) { - err = "repl-ping-slave-period must be 1 or greater"; + err = "repl-ping-replica-period must be 1 or greater"; goto loaderr; } } else if (!strcasecmp(argv[0],"repl-timeout") && argc == 2) { @@ -391,15 +395,24 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { zfree(server.masterauth); server.masterauth = argv[1][0] ? zstrdup(argv[1]) : NULL; - } else if (!strcasecmp(argv[0],"slave-serve-stale-data") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-serve-stale-data") || + !strcasecmp(argv[0],"replica-serve-stale-data")) + && argc == 2) + { if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-read-only") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-read-only") || + !strcasecmp(argv[0],"replica-read-only")) + && argc == 2) + { if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-ignore-maxmemory") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-ignore-maxmemory") || + !strcasecmp(argv[0],"replica-ignore-maxmemory")) + && argc == 2) + { if ((server.repl_slave_ignore_maxmemory = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } @@ -427,7 +440,9 @@ void loadServerConfigFromString(char *config) { if ((server.lazyfree_lazy_server_del = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-lazy-flush") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-lazy-flush") || + !strcasecmp(argv[0],"replica-lazy-flush")) && argc == 2) + { if ((server.repl_slave_lazy_flush = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } @@ -659,15 +674,17 @@ void loadServerConfigFromString(char *config) { err = "cluster migration barrier must zero or positive"; goto loaderr; } - } else if (!strcasecmp(argv[0],"cluster-slave-validity-factor") + } else if ((!strcasecmp(argv[0],"cluster-slave-validity-factor") || + !strcasecmp(argv[0],"cluster-replica-validity-factor")) && argc == 2) { server.cluster_slave_validity_factor = atoi(argv[1]); if (server.cluster_slave_validity_factor < 0) { - err = "cluster slave validity factor must be zero or positive"; + err = "cluster replica validity factor must be zero or positive"; goto loaderr; } - } else if (!strcasecmp(argv[0],"cluster-slave-no-failover") && + } else if ((!strcasecmp(argv[0],"cluster-slave-no-failover") || + !strcasecmp(argv[0],"cluster-replica-no-failiver")) && argc == 2) { server.cluster_slave_no_failover = yesnotoi(argv[1]); @@ -720,27 +737,37 @@ void loadServerConfigFromString(char *config) { if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-priority") || + !strcasecmp(argv[0],"replica-priority")) && argc == 2) + { server.slave_priority = atoi(argv[1]); - } else if (!strcasecmp(argv[0],"slave-announce-ip") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-announce-ip") || + !strcasecmp(argv[0],"replica-announce-ip")) && argc == 2) + { zfree(server.slave_announce_ip); server.slave_announce_ip = zstrdup(argv[1]); - } else if (!strcasecmp(argv[0],"slave-announce-port") && argc == 2) { + } else if ((!strcasecmp(argv[0],"slave-announce-port") || + !strcasecmp(argv[0],"replica-announce-port")) && argc == 2) + { server.slave_announce_port = atoi(argv[1]); if (server.slave_announce_port < 0 || server.slave_announce_port > 65535) { err = "Invalid port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"min-slaves-to-write") && argc == 2) { + } else if ((!strcasecmp(argv[0],"min-slaves-to-write") || + !strcasecmp(argv[0],"min-replicas-to-write")) && argc == 2) + { server.repl_min_slaves_to_write = atoi(argv[1]); if (server.repl_min_slaves_to_write < 0) { - err = "Invalid value for min-slaves-to-write."; goto loaderr; + err = "Invalid value for min-replicas-to-write."; goto loaderr; } - } else if (!strcasecmp(argv[0],"min-slaves-max-lag") && argc == 2) { + } else if ((!strcasecmp(argv[0],"min-slaves-max-lag") || + !strcasecmp(argv[0],"min-replicas-max-lag")) && argc == 2) + { server.repl_min_slaves_max_lag = atoi(argv[1]); if (server.repl_min_slaves_max_lag < 0) { - err = "Invalid value for min-slaves-max-lag."; goto loaderr; + err = "Invalid value for min-replicas-max-lag."; goto loaderr; } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -782,7 +809,7 @@ void loadServerConfigFromString(char *config) { if (server.cluster_enabled && server.masterhost) { linenum = slaveof_linenum; i = linenum-1; - err = "slaveof directive not allowed in cluster mode"; + err = "replicaof directive not allowed in cluster mode"; goto loaderr; } @@ -866,6 +893,10 @@ void loadServerConfig(char *filename, char *options) { #define config_set_special_field(_name) \ } else if (!strcasecmp(c->argv[2]->ptr,_name)) { +#define config_set_special_field_with_alias(_name1,_name2) \ + } else if (!strcasecmp(c->argv[2]->ptr,_name1) || \ + !strcasecmp(c->argv[2]->ptr,_name2)) { + #define config_set_else } else void configSetCommand(client *c) { @@ -1025,7 +1056,9 @@ void configSetCommand(client *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } config_set_special_field("slave-announce-ip") { + } config_set_special_field_with_alias("slave-announce-ip", + "replica-announce-ip") + { zfree(server.slave_announce_ip); server.slave_announce_ip = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; @@ -1041,6 +1074,8 @@ void configSetCommand(client *c) { "cluster-require-full-coverage",server.cluster_require_full_coverage) { } config_set_bool_field( "cluster-slave-no-failover",server.cluster_slave_no_failover) { + } config_set_bool_field( + "cluster-replica-no-failover",server.cluster_slave_no_failover) { } config_set_bool_field( "aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync) { } config_set_bool_field( @@ -1051,10 +1086,16 @@ void configSetCommand(client *c) { "aof-use-rdb-preamble",server.aof_use_rdb_preamble) { } config_set_bool_field( "slave-serve-stale-data",server.repl_serve_stale_data) { + } config_set_bool_field( + "replica-serve-stale-data",server.repl_serve_stale_data) { } config_set_bool_field( "slave-read-only",server.repl_slave_ro) { + } config_set_bool_field( + "replica-read-only",server.repl_slave_ro) { } config_set_bool_field( "slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { + } config_set_bool_field( + "replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory) { } config_set_bool_field( "activerehashing",server.activerehashing) { } config_set_bool_field( @@ -1082,6 +1123,8 @@ void configSetCommand(client *c) { "lazyfree-lazy-server-del",server.lazyfree_lazy_server_del) { } config_set_bool_field( "slave-lazy-flush",server.repl_slave_lazy_flush) { + } config_set_bool_field( + "replica-lazy-flush",server.repl_slave_lazy_flush) { } config_set_bool_field( "no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite) { } config_set_bool_field( @@ -1145,6 +1188,8 @@ void configSetCommand(client *c) { "latency-monitor-threshold",server.latency_monitor_threshold,0,LLONG_MAX){ } config_set_numerical_field( "repl-ping-slave-period",server.repl_ping_slave_period,1,INT_MAX) { + } config_set_numerical_field( + "repl-ping-replica-period",server.repl_ping_slave_period,1,INT_MAX) { } config_set_numerical_field( "repl-timeout",server.repl_timeout,1,INT_MAX) { } config_set_numerical_field( @@ -1171,6 +1216,8 @@ void configSetCommand(client *c) { "cluster-migration-barrier",server.cluster_migration_barrier,0,INT_MAX){ } config_set_numerical_field( "cluster-slave-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { + } config_set_numerical_field( + "cluster-replica-validity-factor",server.cluster_slave_validity_factor,0,INT_MAX) { } config_set_numerical_field( "hz",server.config_hz,0,INT_MAX) { /* Hz is more an hint from the user, so we accept values out of range @@ -1282,6 +1329,7 @@ void configGetCommand(client *c) { config_get_string_field("logfile",server.logfile); config_get_string_field("pidfile",server.pidfile); config_get_string_field("slave-announce-ip",server.slave_announce_ip); + config_get_string_field("replica-announce-ip",server.slave_announce_ip); /* Numerical values */ config_get_numerical_field("maxmemory",server.maxmemory); @@ -1334,19 +1382,25 @@ void configGetCommand(client *c) { config_get_numerical_field("tcp-backlog",server.tcp_backlog); config_get_numerical_field("databases",server.dbnum); config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); + config_get_numerical_field("repl-ping-replica-period",server.repl_ping_slave_period); config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); config_get_numerical_field("slave-priority",server.slave_priority); + config_get_numerical_field("replica-priority",server.slave_priority); config_get_numerical_field("slave-announce-port",server.slave_announce_port); + config_get_numerical_field("replica-announce-port",server.slave_announce_port); config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); + config_get_numerical_field("min-replicas-to-write",server.repl_min_slaves_to_write); config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); + config_get_numerical_field("min-replicas-max-lag",server.repl_min_slaves_max_lag); config_get_numerical_field("hz",server.config_hz); config_get_numerical_field("cluster-node-timeout",server.cluster_node_timeout); config_get_numerical_field("cluster-migration-barrier",server.cluster_migration_barrier); config_get_numerical_field("cluster-slave-validity-factor",server.cluster_slave_validity_factor); + config_get_numerical_field("cluster-replica-validity-factor",server.cluster_slave_validity_factor); config_get_numerical_field("repl-diskless-sync-delay",server.repl_diskless_sync_delay); config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); @@ -1355,14 +1409,22 @@ void configGetCommand(client *c) { server.cluster_require_full_coverage); config_get_bool_field("cluster-slave-no-failover", server.cluster_slave_no_failover); + config_get_bool_field("cluster-replica-no-failover", + server.cluster_slave_no_failover); config_get_bool_field("no-appendfsync-on-rewrite", server.aof_no_fsync_on_rewrite); config_get_bool_field("slave-serve-stale-data", server.repl_serve_stale_data); + config_get_bool_field("replica-serve-stale-data", + server.repl_serve_stale_data); config_get_bool_field("slave-read-only", server.repl_slave_ro); + config_get_bool_field("replica-read-only", + server.repl_slave_ro); config_get_bool_field("slave-ignore-maxmemory", server.repl_slave_ignore_maxmemory); + config_get_bool_field("replica-ignore-maxmemory", + server.repl_slave_ignore_maxmemory); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); @@ -1391,6 +1453,8 @@ void configGetCommand(client *c) { server.lazyfree_lazy_server_del); config_get_bool_field("slave-lazy-flush", server.repl_slave_lazy_flush); + config_get_bool_field("replica-lazy-flush", + server.repl_slave_lazy_flush); config_get_bool_field("dynamic-hz", server.dynamic_hz); @@ -1464,10 +1528,14 @@ void configGetCommand(client *c) { addReplyBulkCString(c,buf); matches++; } - if (stringmatch(pattern,"slaveof",1)) { + if (stringmatch(pattern,"slaveof",1) || + stringmatch(pattern,"replicaof",1)) + { + char *optname = stringmatch(pattern,"slaveof",1) ? + "slaveof" : "replicaof"; char buf[256]; - addReplyBulkCString(c,"slaveof"); + addReplyBulkCString(c,optname); if (server.masterhost) snprintf(buf,sizeof(buf),"%s %d", server.masterhost, server.masterport); @@ -1623,8 +1691,20 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { /* Now we populate the state according to the content of this line. * Append the line and populate the option -> line numbers map. */ rewriteConfigAppendLine(state,line); - rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + /* Translate options using the word "slave" to the corresponding name + * "replica", before adding such option to the config name -> lines + * mapping. */ + char *p = strstr(argv[0],"slave"); + if (p) { + sds alt = sdsempty(); + alt = sdscatlen(alt,argv[0],p-argv[0]);; + alt = sdscatlen(alt,"replica",7); + alt = sdscatlen(alt,p+5,strlen(p+5)); + sdsfree(argv[0]); + argv[0] = alt; + } + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); sdsfreesplitres(argv,argc); } fclose(fp); @@ -1811,15 +1891,14 @@ void rewriteConfigDirOption(struct rewriteConfigState *state) { } /* Rewrite the slaveof option. */ -void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { - char *option = "slaveof"; +void rewriteConfigSlaveofOption(struct rewriteConfigState *state, char *option) { sds line; /* If this is a master, we want all the slaveof config options * in the file to be removed. Note that if this is a cluster instance * we don't want a slaveof directive inside redis.conf. */ if (server.cluster_enabled || server.masterhost == NULL) { - rewriteConfigMarkAsProcessed(state,"slaveof"); + rewriteConfigMarkAsProcessed(state,option); return; } line = sdscatprintf(sdsempty(),"%s %s %d", option, @@ -2040,7 +2119,7 @@ int rewriteConfig(char *path) { rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,CONFIG_DEFAULT_UNIX_SOCKET_PERM); rewriteConfigNumericalOption(state,"timeout",server.maxidletime,CONFIG_DEFAULT_CLIENT_TIMEOUT); rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,CONFIG_DEFAULT_TCP_KEEPALIVE); - rewriteConfigNumericalOption(state,"slave-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); + rewriteConfigNumericalOption(state,"replica-announce-port",server.slave_announce_port,CONFIG_DEFAULT_SLAVE_ANNOUNCE_PORT); rewriteConfigEnumOption(state,"loglevel",server.verbosity,loglevel_enum,CONFIG_DEFAULT_VERBOSITY); rewriteConfigStringOption(state,"logfile",server.logfile,CONFIG_DEFAULT_LOGFILE); rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,CONFIG_DEFAULT_SYSLOG_ENABLED); @@ -2053,23 +2132,23 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,CONFIG_DEFAULT_RDB_CHECKSUM); rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,CONFIG_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); - rewriteConfigSlaveofOption(state); - rewriteConfigStringOption(state,"slave-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); + rewriteConfigSlaveofOption(state,"replicaof"); + rewriteConfigStringOption(state,"replica-announce-ip",server.slave_announce_ip,CONFIG_DEFAULT_SLAVE_ANNOUNCE_IP); rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); rewriteConfigStringOption(state,"cluster-announce-ip",server.cluster_announce_ip,NULL); - rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); - rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); - rewriteConfigYesNoOption(state,"slave-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); - rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); + rewriteConfigYesNoOption(state,"replica-serve-stale-data",server.repl_serve_stale_data,CONFIG_DEFAULT_SLAVE_SERVE_STALE_DATA); + rewriteConfigYesNoOption(state,"replica-read-only",server.repl_slave_ro,CONFIG_DEFAULT_SLAVE_READ_ONLY); + rewriteConfigYesNoOption(state,"replica-ignore-maxmemory",server.repl_slave_ignore_maxmemory,CONFIG_DEFAULT_SLAVE_IGNORE_MAXMEMORY); + rewriteConfigNumericalOption(state,"repl-ping-replica-period",server.repl_ping_slave_period,CONFIG_DEFAULT_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,CONFIG_DEFAULT_REPL_TIMEOUT); rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,CONFIG_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,CONFIG_DEFAULT_REPL_BACKLOG_TIME_LIMIT); rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,CONFIG_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigYesNoOption(state,"repl-diskless-sync",server.repl_diskless_sync,CONFIG_DEFAULT_REPL_DISKLESS_SYNC); rewriteConfigNumericalOption(state,"repl-diskless-sync-delay",server.repl_diskless_sync_delay,CONFIG_DEFAULT_REPL_DISKLESS_SYNC_DELAY); - rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); - rewriteConfigNumericalOption(state,"min-slaves-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); - rewriteConfigNumericalOption(state,"min-slaves-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); + rewriteConfigNumericalOption(state,"replica-priority",server.slave_priority,CONFIG_DEFAULT_SLAVE_PRIORITY); + rewriteConfigNumericalOption(state,"min-replicas-to-write",server.repl_min_slaves_to_write,CONFIG_DEFAULT_MIN_SLAVES_TO_WRITE); + rewriteConfigNumericalOption(state,"min-replicas-max-lag",server.repl_min_slaves_max_lag,CONFIG_DEFAULT_MIN_SLAVES_MAX_LAG); rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,CONFIG_DEFAULT_MAX_CLIENTS); rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,CONFIG_DEFAULT_MAXMEMORY); @@ -2095,10 +2174,10 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,CONFIG_DEFAULT_CLUSTER_CONFIG_FILE); rewriteConfigYesNoOption(state,"cluster-require-full-coverage",server.cluster_require_full_coverage,CLUSTER_DEFAULT_REQUIRE_FULL_COVERAGE); - rewriteConfigYesNoOption(state,"cluster-slave-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); + rewriteConfigYesNoOption(state,"cluster-replica-no-failover",server.cluster_slave_no_failover,CLUSTER_DEFAULT_SLAVE_NO_FAILOVER); rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,CLUSTER_DEFAULT_NODE_TIMEOUT); rewriteConfigNumericalOption(state,"cluster-migration-barrier",server.cluster_migration_barrier,CLUSTER_DEFAULT_MIGRATION_BARRIER); - rewriteConfigNumericalOption(state,"cluster-slave-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); + rewriteConfigNumericalOption(state,"cluster-replica-validity-factor",server.cluster_slave_validity_factor,CLUSTER_DEFAULT_SLAVE_VALIDITY); rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,CONFIG_DEFAULT_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"latency-monitor-threshold",server.latency_monitor_threshold,CONFIG_DEFAULT_LATENCY_MONITOR_THRESHOLD); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,CONFIG_DEFAULT_SLOWLOG_MAX_LEN); @@ -2126,7 +2205,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"lazyfree-lazy-eviction",server.lazyfree_lazy_eviction,CONFIG_DEFAULT_LAZYFREE_LAZY_EVICTION); rewriteConfigYesNoOption(state,"lazyfree-lazy-expire",server.lazyfree_lazy_expire,CONFIG_DEFAULT_LAZYFREE_LAZY_EXPIRE); rewriteConfigYesNoOption(state,"lazyfree-lazy-server-del",server.lazyfree_lazy_server_del,CONFIG_DEFAULT_LAZYFREE_LAZY_SERVER_DEL); - rewriteConfigYesNoOption(state,"slave-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); + rewriteConfigYesNoOption(state,"replica-lazy-flush",server.repl_slave_lazy_flush,CONFIG_DEFAULT_SLAVE_LAZY_FLUSH); rewriteConfigYesNoOption(state,"dynamic-hz",server.dynamic_hz,CONFIG_DEFAULT_DYNAMIC_HZ); /* Rewrite Sentinel config if in Sentinel mode. */ From dcdcbcc3754f130a114ad716726e526ba252918e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:27:37 +0200 Subject: [PATCH 130/616] Slave removal: slave -> replica in redis.conf and output buffer option. --- redis.conf | 256 +++++++++++++++++++++++------------------------ src/config.c | 4 +- src/networking.c | 1 + 3 files changed, 132 insertions(+), 129 deletions(-) diff --git a/redis.conf b/redis.conf index 886c64f21..868f14c48 100644 --- a/redis.conf +++ b/redis.conf @@ -264,59 +264,59 @@ dir ./ ################################# REPLICATION ################################# -# Master-Slave replication. Use slaveof to make a Redis instance a copy of +# Master-Replica replication. Use replcaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # # 1) Redis replication is asynchronous, but you can configure a master to # stop accepting writes if it appears to be not connected with at least -# a given number of slaves. -# 2) Redis slaves are able to perform a partial resynchronization with the +# a given number of replicas. +# 2) Redis replicas are able to perform a partial resynchronization with the # master if the replication link is lost for a relatively small amount of # time. You may want to configure the replication backlog size (see the next # sections of this file) with a sensible value depending on your needs. # 3) Replication is automatic and does not need user intervention. After a -# network partition slaves automatically try to reconnect to masters +# network partition replicas automatically try to reconnect to masters # and resynchronize with them. # -# slaveof +# replicaof # If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before +# directive below) it is possible to tell the replica to authenticate before # starting the replication synchronization process, otherwise the master will -# refuse the slave request. +# refuse the replica request. # # masterauth -# When a slave loses its connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: +# When a replica loses its connection with the master, or when the replication +# is still in progress, the replica can act in two different ways: # -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will +# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale-data is set to 'no' the slave will reply with +# 2) if replica-serve-stale-data is set to 'no' the replica will reply with # an error "SYNC with master in progress" to all the kind of commands -# but to INFO, SLAVEOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, +# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, # SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, # COMMAND, POST, HOST: and LATENCY. # -slave-serve-stale-data yes +replica-serve-stale-data yes -# You can configure a slave instance to accept writes or not. Writing against -# a slave instance may be useful to store some ephemeral data (because data -# written on a slave will be easily deleted after resync with the master) but +# You can configure a replica instance to accept writes or not. Writing against +# a replica instance may be useful to store some ephemeral data (because data +# written on a replica will be easily deleted after resync with the master) but # may also cause problems if clients are writing to it because of a # misconfiguration. # -# Since Redis 2.6 by default slaves are read-only. +# Since Redis 2.6 by default replicas are read-only. # -# Note: read only slaves are not designed to be exposed to untrusted clients +# Note: read only replicas are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. -# Still a read only slave exports by default all the administrative commands +# Still a read only replica exports by default all the administrative commands # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only slaves using 'rename-command' to shadow all the +# security of read only replicas using 'rename-command' to shadow all the # administrative / dangerous commands. -slave-read-only yes +replica-read-only yes # Replication SYNC strategy: disk or socket. # @@ -324,25 +324,25 @@ slave-read-only yes # WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY # ------------------------------------------------------- # -# New slaves and reconnecting slaves that are not able to continue the replication +# New replicas and reconnecting replicas that are not able to continue the replication # process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the slaves. +# synchronization". An RDB file is transmitted from the master to the replicas. # The transmission can happen in two different ways: # # 1) Disk-backed: The Redis master creates a new process that writes the RDB # file on disk. Later the file is transferred by the parent -# process to the slaves incrementally. +# process to the replicas incrementally. # 2) Diskless: The Redis master creates a new process that directly writes the -# RDB file to slave sockets, without touching the disk at all. +# RDB file to replica sockets, without touching the disk at all. # -# With disk-backed replication, while the RDB file is generated, more slaves +# With disk-backed replication, while the RDB file is generated, more replicas # can be queued and served with the RDB file as soon as the current child producing # the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new slaves arriving will be queued and a new transfer +# the transfer starts, new replicas arriving will be queued and a new transfer # will start when the current one terminates. # # When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple slaves +# time (in seconds) before starting the transfer in the hope that multiple replicas # will arrive and the transfer can be parallelized. # # With slow disks and fast (large bandwidth) networks, diskless replication @@ -351,140 +351,140 @@ repl-diskless-sync no # When diskless replication is enabled, it is possible to configure the delay # the server waits in order to spawn the child that transfers the RDB via socket -# to the slaves. +# to the replicas. # # This is important since once the transfer starts, it is not possible to serve -# new slaves arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more slaves arrive. +# new replicas arriving, that will be queued for the next RDB transfer, so the server +# waits a delay in order to let more replicas arrive. # # The delay is specified in seconds, and by default is 5 seconds. To disable # it entirely just set it to 0 seconds and the transfer will start ASAP. repl-diskless-sync-delay 5 -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 +# replicas send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_replica_period option. The default value is 10 # seconds. # -# repl-ping-slave-period 10 +# repl-ping-replica-period 10 # The following option sets the replication timeout for: # -# 1) Bulk transfer I/O during SYNC, from the point of view of slave. -# 2) Master timeout from the point of view of slaves (data, pings). -# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). +# 1) Bulk transfer I/O during SYNC, from the point of view of replica. +# 2) Master timeout from the point of view of replicas (data, pings). +# 3) replica timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. +# specified for repl-ping-replica-period otherwise a timeout will be detected +# every time there is low traffic between the master and the replica. # # repl-timeout 60 -# Disable TCP_NODELAY on the slave socket after SYNC? +# Disable TCP_NODELAY on the replica socket after SYNC? # # If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to slaves. But this can add a delay for -# the data to appear on the slave side, up to 40 milliseconds with +# less bandwidth to send data to replicas. But this can add a delay for +# the data to appear on the replica side, up to 40 milliseconds with # Linux kernels using a default configuration. # -# If you select "no" the delay for data to appear on the slave side will +# If you select "no" the delay for data to appear on the replica side will # be reduced but more bandwidth will be used for replication. # # By default we optimize for low latency, but in very high traffic conditions -# or when the master and slaves are many hops away, turning this to "yes" may +# or when the master and replicas are many hops away, turning this to "yes" may # be a good idea. repl-disable-tcp-nodelay no # Set the replication backlog size. The backlog is a buffer that accumulates -# slave data when slaves are disconnected for some time, so that when a slave +# replica data when replicas are disconnected for some time, so that when a replica # wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the slave missed while +# resync is enough, just passing the portion of data the replica missed while # disconnected. # -# The bigger the replication backlog, the longer the time the slave can be +# The bigger the replication backlog, the longer the time the replica can be # disconnected and later be able to perform a partial resynchronization. # -# The backlog is only allocated once there is at least a slave connected. +# The backlog is only allocated once there is at least a replica connected. # # repl-backlog-size 1mb -# After a master has no longer connected slaves for some time, the backlog +# After a master has no longer connected replicas for some time, the backlog # will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last slave disconnected, for +# need to elapse, starting from the time the last replica disconnected, for # the backlog buffer to be freed. # -# Note that slaves never free the backlog for timeout, since they may be +# Note that replicas never free the backlog for timeout, since they may be # promoted to masters later, and should be able to correctly "partially -# resynchronize" with the slaves: hence they should always accumulate backlog. +# resynchronize" with the replicas: hence they should always accumulate backlog. # # A value of 0 means to never release the backlog. # # repl-backlog-ttl 3600 -# The slave priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a slave to promote into a +# The replica priority is an integer number published by Redis in the INFO output. +# It is used by Redis Sentinel in order to select a replica to promote into a # master if the master is no longer working correctly. # -# A slave with a low priority number is considered better for promotion, so -# for instance if there are three slaves with priority 10, 100, 25 Sentinel will +# A replica with a low priority number is considered better for promotion, so +# for instance if there are three replicas with priority 10, 100, 25 Sentinel will # pick the one with priority 10, that is the lowest. # -# However a special priority of 0 marks the slave as not able to perform the -# role of master, so a slave with priority of 0 will never be selected by +# However a special priority of 0 marks the replica as not able to perform the +# role of master, so a replica with priority of 0 will never be selected by # Redis Sentinel for promotion. # # By default the priority is 100. -slave-priority 100 +replica-priority 100 # It is possible for a master to stop accepting writes if there are less than -# N slaves connected, having a lag less or equal than M seconds. +# N replicas connected, having a lag less or equal than M seconds. # -# The N slaves need to be in "online" state. +# The N replicas need to be in "online" state. # # The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the slave, that is usually sent every second. +# the last ping received from the replica, that is usually sent every second. # # This option does not GUARANTEE that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough slaves +# will limit the window of exposure for lost writes in case not enough replicas # are available, to the specified number of seconds. # -# For example to require at least 3 slaves with a lag <= 10 seconds use: +# For example to require at least 3 replicas with a lag <= 10 seconds use: # -# min-slaves-to-write 3 -# min-slaves-max-lag 10 +# min-replicas-to-write 3 +# min-replicas-max-lag 10 # # Setting one or the other to 0 disables the feature. # -# By default min-slaves-to-write is set to 0 (feature disabled) and -# min-slaves-max-lag is set to 10. +# By default min-replicas-to-write is set to 0 (feature disabled) and +# min-replicas-max-lag is set to 10. # A Redis master is able to list the address and port of the attached -# slaves in different ways. For example the "INFO replication" section +# replicas in different ways. For example the "INFO replication" section # offers this information, which is used, among other tools, by -# Redis Sentinel in order to discover slave instances. +# Redis Sentinel in order to discover replica instances. # Another place where this info is available is in the output of the # "ROLE" command of a master. # -# The listed IP and address normally reported by a slave is obtained +# The listed IP and address normally reported by a replica is obtained # in the following way: # # IP: The address is auto detected by checking the peer address -# of the socket used by the slave to connect with the master. +# of the socket used by the replica to connect with the master. # -# Port: The port is communicated by the slave during the replication -# handshake, and is normally the port that the slave is using to +# Port: The port is communicated by the replica during the replication +# handshake, and is normally the port that the replica is using to # list for connections. # # However when port forwarding or Network Address Translation (NAT) is -# used, the slave may be actually reachable via different IP and port -# pairs. The following two options can be used by a slave in order to +# used, the replica may be actually reachable via different IP and port +# pairs. The following two options can be used by a replica in order to # report to its master a specific set of IP and port, so that both INFO # and ROLE will report those values. # # There is no need to use both the options if you need to override just # the port or the IP address. # -# slave-announce-ip 5.5.5.5 -# slave-announce-port 1234 +# replica-announce-ip 5.5.5.5 +# replica-announce-port 1234 ################################## SECURITY ################################### @@ -518,7 +518,7 @@ slave-priority 100 # rename-command CONFIG "" # # Please note that changing the name of commands that are logged into the -# AOF file or transmitted to slaves may cause problems. +# AOF file or transmitted to replicas may cause problems. ################################### CLIENTS #################################### @@ -547,15 +547,15 @@ slave-priority 100 # This option is usually useful when using Redis as an LRU or LFU cache, or to # set a hard memory limit for an instance (using the 'noeviction' policy). # -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted +# WARNING: If you have replicas attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the replicas are subtracted # from the used memory count, so that network problems / resyncs will # not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion +# buffer of replicas is full with DELs of keys evicted triggering the deletion # of more keys, and so forth until the database is completely emptied. # -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave +# In short... if you have replicas attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # # maxmemory @@ -602,25 +602,25 @@ slave-priority 100 # # maxmemory-samples 5 -# Starting from Redis 5, by default a slave will ignore its maxmemory setting +# Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means # that the eviction of keys will be just handled by the master, sending the -# DEL commands to the slave as keys evict in the master side. +# DEL commands to the replica as keys evict in the master side. # -# This behavior ensures that masters and slaves stay consistent, and is usually -# what you want, however if your slave is writable, or you want the slave to have +# This behavior ensures that masters and replicas stay consistent, and is usually +# what you want, however if your replica is writable, or you want the replica to have # a different memory setting, and you are sure all the writes performed to the -# slave are idempotent, then you may change this default (but be sure to understand +# replica are idempotent, then you may change this default (but be sure to understand # what you are doing). # -# Note that since the slave by default does not evict, it may end using more +# Note that since the replica by default does not evict, it may end using more # memory than the one set via maxmemory (there are certain buffers that may -# be larger on the slave, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your slaves and make sure they have enough +# be larger on the replica, or data structures may sometimes take more memory and so +# forth). So make sure you monitor your replicas and make sure they have enough # memory to never hit a real out-of-memory condition before the master hits # the configured maxmemory setting. # -# slave-ingore-maxmemory yes +# replica-ingore-maxmemory yes ############################# LAZY FREEING #################################### @@ -657,7 +657,7 @@ slave-priority 100 # or SORT with STORE option may delete existing keys. The SET command # itself removes any old content of the specified key in order to replace # it with the specified string. -# 4) During replication, when a slave performs a full resynchronization with +# 4) During replication, when a replica performs a full resynchronization with # its master, the content of the whole database is removed in order to # load the RDB file just transferred. # @@ -669,7 +669,7 @@ slave-priority 100 lazyfree-lazy-eviction no lazyfree-lazy-expire no lazyfree-lazy-server-del no -slave-lazy-flush no +replica-lazy-flush no ############################## APPEND ONLY MODE ############################### @@ -846,42 +846,42 @@ lua-time-limit 5000 # # cluster-node-timeout 15000 -# A slave of a failing master will avoid to start a failover if its data +# A replica of a failing master will avoid to start a failover if its data # looks too old. # -# There is no simple way for a slave to actually have an exact measure of +# There is no simple way for a replica to actually have an exact measure of # its "data age", so the following two checks are performed: # -# 1) If there are multiple slaves able to failover, they exchange messages -# in order to try to give an advantage to the slave with the best +# 1) If there are multiple replicas able to failover, they exchange messages +# in order to try to give an advantage to the replica with the best # replication offset (more data from the master processed). -# Slaves will try to get their rank by offset, and apply to the start +# replicas will try to get their rank by offset, and apply to the start # of the failover a delay proportional to their rank. # -# 2) Every single slave computes the time of the last interaction with +# 2) Every single replica computes the time of the last interaction with # its master. This can be the last ping or command received (if the master # is still in the "connected" state), or the time that elapsed since the # disconnection with the master (if the replication link is currently down). -# If the last interaction is too old, the slave will not try to failover +# If the last interaction is too old, the replica will not try to failover # at all. # -# The point "2" can be tuned by user. Specifically a slave will not perform +# The point "2" can be tuned by user. Specifically a replica will not perform # the failover if, since the last interaction with the master, the time # elapsed is greater than: # -# (node-timeout * slave-validity-factor) + repl-ping-slave-period +# (node-timeout * replica-validity-factor) + repl-ping-replica-period # -# So for example if node-timeout is 30 seconds, and the slave-validity-factor -# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the -# slave will not try to failover if it was not able to talk with the master +# So for example if node-timeout is 30 seconds, and the replica-validity-factor +# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the +# replica will not try to failover if it was not able to talk with the master # for longer than 310 seconds. # -# A large slave-validity-factor may allow slaves with too old data to failover +# A large replica-validity-factor may allow replicas with too old data to failover # a master, while a too small value may prevent the cluster from being able to -# elect a slave at all. +# elect a replica at all. # -# For maximum availability, it is possible to set the slave-validity-factor -# to a value of 0, which means, that slaves will always try to failover the +# For maximum availability, it is possible to set the replica-validity-factor +# to a value of 0, which means, that replicas will always try to failover the # master regardless of the last time they interacted with the master. # (However they'll always try to apply a delay proportional to their # offset rank). @@ -889,22 +889,22 @@ lua-time-limit 5000 # Zero is the only value able to guarantee that when all the partitions heal # the cluster will always be able to continue. # -# cluster-slave-validity-factor 10 +# cluster-replica-validity-factor 10 -# Cluster slaves are able to migrate to orphaned masters, that are masters -# that are left without working slaves. This improves the cluster ability +# Cluster replicas are able to migrate to orphaned masters, that are masters +# that are left without working replicas. This improves the cluster ability # to resist to failures as otherwise an orphaned master can't be failed over -# in case of failure if it has no working slaves. +# in case of failure if it has no working replicas. # -# Slaves migrate to orphaned masters only if there are still at least a -# given number of other working slaves for their old master. This number -# is the "migration barrier". A migration barrier of 1 means that a slave -# will migrate only if there is at least 1 other working slave for its master -# and so forth. It usually reflects the number of slaves you want for every +# replicas migrate to orphaned masters only if there are still at least a +# given number of other working replicas for their old master. This number +# is the "migration barrier". A migration barrier of 1 means that a replica +# will migrate only if there is at least 1 other working replica for its master +# and so forth. It usually reflects the number of replicas you want for every # master in your cluster. # -# Default is 1 (slaves migrate only if their masters remain with at least -# one slave). To disable migration just set it to a very large value. +# Default is 1 (replicas migrate only if their masters remain with at least +# one replica). To disable migration just set it to a very large value. # A value of 0 can be set but is useful only for debugging and dangerous # in production. # @@ -923,7 +923,7 @@ lua-time-limit 5000 # # cluster-require-full-coverage yes -# This option, when set to yes, prevents slaves from trying to failover its +# This option, when set to yes, prevents replicas from trying to failover its # master during master failures. However the master can still perform a # manual failover, if forced to do so. # @@ -931,7 +931,7 @@ lua-time-limit 5000 # data center operations, where we want one side to never be promoted if not # in the case of a total DC failure. # -# cluster-slave-no-failover no +# cluster-replica-no-failover no # In order to setup your cluster make sure to read the documentation # available at http://redis.io web site. @@ -1165,7 +1165,7 @@ activerehashing yes # The limit can be set differently for the three different classes of clients: # # normal -> normal clients including MONITOR clients -# slave -> slave clients +# replica -> replica clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: @@ -1186,12 +1186,12 @@ activerehashing yes # asynchronous clients may create a scenario where data is requested faster # than it can read. # -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. +# Instead there is a default limit for pubsub and replica clients, since +# subscribers and replicas receive data in a push fashion. # # Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit replica 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 # Client query buffers accumulate new commands. They are limited to a fixed diff --git a/src/config.c b/src/config.c index 3b38f98d8..2feb39336 100644 --- a/src/config.c +++ b/src/config.c @@ -1940,8 +1940,10 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state rewriteConfigFormatMemory(soft,sizeof(soft), server.client_obuf_limits[j].soft_limit_bytes); + char *typename = getClientTypeName(j); + if (!strcmp(typename,"slave")) typename = "replica"; line = sdscatprintf(sdsempty(),"%s %s %s %s %ld", - option, getClientTypeName(j), hard, soft, + option, typename, hard, soft, (long) server.client_obuf_limits[j].soft_limit_seconds); rewriteConfigRewriteLine(state,option,line,force); } diff --git a/src/networking.c b/src/networking.c index 01307c7a3..df670b7c7 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1991,6 +1991,7 @@ int getClientType(client *c) { int getClientTypeByName(char *name) { if (!strcasecmp(name,"normal")) return CLIENT_TYPE_NORMAL; else if (!strcasecmp(name,"slave")) return CLIENT_TYPE_SLAVE; + else if (!strcasecmp(name,"replica")) return CLIENT_TYPE_SLAVE; else if (!strcasecmp(name,"pubsub")) return CLIENT_TYPE_PUBSUB; else if (!strcasecmp(name,"master")) return CLIENT_TYPE_MASTER; else return -1; From 922d1d23878eebe6ac114dc30e1c4ebb32a88743 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 12:56:02 +0200 Subject: [PATCH 131/616] Slave removal: fix typo of replicaof. --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 868f14c48..66ad3418a 100644 --- a/redis.conf +++ b/redis.conf @@ -264,7 +264,7 @@ dir ./ ################################# REPLICATION ################################# -# Master-Replica replication. Use replcaof to make a Redis instance a copy of +# Master-Replica replication. Use replicaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # # 1) Redis replication is asynchronous, but you can configure a master to From 92eb72d175e13f44afe32fa624dc3833fd3654b8 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:37:23 +0200 Subject: [PATCH 132/616] Slave removal: slave mode -> replica mode text in redis-cli. --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0a3d78db3..c9e8d26b3 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1154,7 +1154,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) { } if (config.slave_mode) { - printf("Entering slave output mode... (press Ctrl-C to quit)\n"); + printf("Entering replica output mode... (press Ctrl-C to quit)\n"); slaveMode(); config.slave_mode = 0; zfree(argvlen); From 22e38e63ffa5234d3e934e0c94298712ab9a7024 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:39:51 +0200 Subject: [PATCH 133/616] Slave removal: Make obvious in redis.conf what a replica is. --- redis.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/redis.conf b/redis.conf index 66ad3418a..3ca9fe5bc 100644 --- a/redis.conf +++ b/redis.conf @@ -267,6 +267,11 @@ dir ./ # Master-Replica replication. Use replicaof to make a Redis instance a copy of # another Redis server. A few things to understand ASAP about Redis replication. # +# +------------------+ +---------------+ +# | Master | ---> | Replica | +# | (receive writes) | | (exact copy) | +# +------------------+ +---------------+ +# # 1) Redis replication is asynchronous, but you can configure a master to # stop accepting writes if it appears to be not connected with at least # a given number of replicas. From 7591c476f89e191cd051aeca91335c24ec5e6b26 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 16:46:14 +0200 Subject: [PATCH 134/616] Slave removal: blocked.c logs fixed. --- src/blocked.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blocked.c b/src/blocked.c index 00212ed69..2b43f2b75 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -215,7 +215,7 @@ void disconnectAllBlockedClients(void) { if (c->flags & CLIENT_BLOCKED) { addReplySds(c,sdsnew( "-UNBLOCKED force unblock from blocking operation, " - "instance state changed (master -> slave?)\r\n")); + "instance state changed (master -> replica?)\r\n")); unblockClient(c); c->flags |= CLIENT_CLOSE_AFTER_REPLY; } From 04d65fc5591933151444ad0adeb736e4599394b0 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 10 Sep 2018 17:02:44 +0200 Subject: [PATCH 135/616] Slave removal: networking.c logs fixed. --- src/networking.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index df670b7c7..06715ee41 100644 --- a/src/networking.c +++ b/src/networking.c @@ -355,8 +355,8 @@ void addReplyErrorLength(client *c, const char *s, size_t len) { * will produce an error. However it is useful to log such events since * they are rare and may hint at errors in a script or a bug in Redis. */ if (c->flags & (CLIENT_MASTER|CLIENT_SLAVE)) { - char* to = c->flags & CLIENT_MASTER? "master": "slave"; - char* from = c->flags & CLIENT_MASTER? "slave": "master"; + char* to = c->flags & CLIENT_MASTER? "master": "replica"; + char* from = c->flags & CLIENT_MASTER? "replica": "master"; char *cmdname = c->lastcmd ? c->lastcmd->name : ""; serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error " "to its %s: '%s' after processing the command " @@ -836,7 +836,7 @@ void freeClient(client *c) { /* Log link disconnection with slave */ if ((c->flags & CLIENT_SLAVE) && !(c->flags & CLIENT_MONITOR)) { - serverLog(LL_WARNING,"Connection with slave %s lost.", + serverLog(LL_WARNING,"Connection with replica %s lost.", replicationGetSlaveName(c)); } @@ -1654,10 +1654,10 @@ void clientCommand(client *c) { "kill -- Kill connection made from .", "kill