From 3e970746491a5a9e1a70804bb49e5527b4aaf17c Mon Sep 17 00:00:00 2001 From: Madelyn Olson <34459052+madolson@users.noreply.github.com> Date: Mon, 19 Apr 2021 22:16:27 -0700 Subject: [PATCH 01/24] Fix memory leak when doing lazyfreeing client tracking table (#8822) Interior rax pointers were not being freed (cherry picked from commit c73b4ddfd96d00ed0d0fde17953ce63d78bc3777) --- src/lazyfree.c | 5 ++--- src/server.h | 1 + tests/unit/tracking.tcl | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/lazyfree.c b/src/lazyfree.c index f18b2027f..a2cf2c3ed 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -39,12 +39,11 @@ void lazyfreeFreeSlotsMap(void *args[]) { atomicIncr(lazyfreed_objects,len); } -/* Release the rax mapping Redis Cluster keys to slots in the - * lazyfree thread. */ +/* Release the key tracking table. */ void lazyFreeTrackingTable(void *args[]) { rax *rt = args[0]; size_t len = rt->numele; - raxFree(rt); + freeTrackingRadixTree(rt); atomicDecr(lazyfree_objects,len); atomicIncr(lazyfreed_objects,len); } diff --git a/src/server.h b/src/server.h index d35eaa425..a2a722c6d 100644 --- a/src/server.h +++ b/src/server.h @@ -1911,6 +1911,7 @@ void disableTracking(client *c); void trackingRememberKeys(client *c); void trackingInvalidateKey(client *c, robj *keyobj); void trackingInvalidateKeysOnFlush(int async); +void freeTrackingRadixTree(rax *rt); void freeTrackingRadixTreeAsync(rax *rt); void trackingLimitUsedSlots(void); uint64_t trackingGetTotalItems(void); diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index 40f1a2a66..4c75b6f48 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -395,6 +395,17 @@ start_server {tags {"tracking network"}} { assert {[lindex msg 2] eq {} } } + test {Test ASYNC flushall} { + clean_all + r CLIENT TRACKING on REDIRECT $redir_id + r GET key1 + r GET key2 + assert_equal [s 0 tracking_total_keys] 2 + $rd_sg FLUSHALL ASYNC + assert_equal [s 0 tracking_total_keys] 0 + assert_equal [lindex [$rd_redirection read] 2] {} + } + # Keys are defined to be evicted 100 at a time by default. # If after eviction the number of keys still surpasses the limit # defined in tracking-table-max-keys, we increases eviction From 5ce259333e8fe944cfc06dc7d1b00fa6cd7a7afc Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Tue, 20 Apr 2021 15:59:44 +0800 Subject: [PATCH 02/24] Fix migrateCommand may migrate wrong value. (#8815) This scene is hard to happen. When first attempt some keys expired, only kv position is updated not ov. Then socket err happens, second attempt is taken. This time kv items may be mismatching with ov items. (cherry picked from commit 080d4579db40d965f8392af5b1da7a99d1a817d5) --- src/cluster.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cluster.c b/src/cluster.c index 0f0ab737e..ba21024be 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5465,9 +5465,10 @@ try_again: if (ttl < 1) ttl = 1; } - /* Relocate valid (non expired) keys into the array in successive + /* Relocate valid (non expired) keys and values into the array in successive * positions to remove holes created by the keys that were present * in the first lookup but are now expired after the second lookup. */ + ov[non_expired] = ov[j]; kv[non_expired++] = kv[j]; serverAssertWithInfo(c,NULL, From d7221e0135cbf9c8cb1c29e671a6050a25302b81 Mon Sep 17 00:00:00 2001 From: bugwz Date: Wed, 21 Apr 2021 02:51:24 +0800 Subject: [PATCH 03/24] Print the number of abnormal line in AOF (#8823) When redis-check-aof finds an error, it prints the line number for faster troubleshooting. (cherry picked from commit 761d7d27711edfbf737def41ff28f5b325fb16c8) --- src/redis-check-aof.c | 6 ++++-- tests/integration/aof.tcl | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index eedb09db5..1507e0a06 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -39,12 +39,14 @@ static char error[1044]; static off_t epos; +static long long line = 1; int consumeNewline(char *buf) { if (strncmp(buf,"\r\n",2) != 0) { ERROR("Expected \\r\\n, got: %02x%02x",buf[0],buf[1]); return 0; } + line += 1; return 1; } @@ -201,8 +203,8 @@ int redis_check_aof_main(int argc, char **argv) { off_t pos = process(fp); off_t diff = size-pos; - printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n", - (long long) size, (long long) pos, (long long) diff); + printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n", + (long long) size, (long long) pos, line, (long long) diff); if (diff > 0) { if (fix) { char buf[2]; diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index e64e2022a..abe2dc10c 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -158,6 +158,18 @@ tags {"aof"} { assert_match "*not valid*" $result } + test "Short read: Utility should show the abnormal line num in AOF" { + create_aof { + append_to_aof [formatCommand set foo hello] + append_to_aof "!!!" + } + + catch { + exec src/redis-check-aof $aof_path + } result + assert_match "*ok_up_to_line=8*" $result + } + test "Short read: Utility should be able to fix the AOF" { set result [exec src/redis-check-aof --fix $aof_path << "y\n"] assert_match "*Successfully truncated AOF*" $result From 3b9ceeb2a4e35f6cef6d731acb5c2e57f1588a6a Mon Sep 17 00:00:00 2001 From: Wang Yuan Date: Thu, 22 Apr 2021 13:32:43 +0800 Subject: [PATCH 04/24] Expire key firstly and then notify keyspace event (#8830) Modules event subscribers may get wrong things in notifyKeyspaceEvent callback, such as wrong number of keys, or be able to lookup this key. This commit changes the order to be like the one in evict.c. Cleanup: Since we know the key exists (it expires now), db*Delete is sure to return 1, so there's no need to check it's output (misleading). (cherry picked from commit 63acfe4b00b9d3e34a53559f965c0bc44c03db61) --- src/db.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index ec68c228c..40377ec3f 100644 --- a/src/db.c +++ b/src/db.c @@ -1541,14 +1541,17 @@ int expireIfNeeded(redisDb *db, robj *key) { if (checkClientPauseTimeoutAndReturnIfPaused()) return 1; /* Delete the key */ + if (server.lazyfree_lazy_expire) { + dbAsyncDelete(db,key); + } else { + dbSyncDelete(db,key); + } server.stat_expiredkeys++; propagateExpire(db,key,server.lazyfree_lazy_expire); notifyKeyspaceEvent(NOTIFY_EXPIRED, "expired",key,db->id); - int retval = server.lazyfree_lazy_expire ? dbAsyncDelete(db,key) : - dbSyncDelete(db,key); - if (retval) signalModifiedKey(NULL,db,key); - return retval; + signalModifiedKey(NULL,db,key); + return 1; } /* ----------------------------------------------------------------------------- From 7ac26c3497789da4365a40d1f920a328c99abc8d Mon Sep 17 00:00:00 2001 From: zyxwvu Shi Date: Thu, 22 Apr 2021 13:59:10 +0800 Subject: [PATCH 05/24] Use monotonic clock to check for Lua script timeout. (#8812) This prevents a case where NTP moves the system clock forward resulting in a false detection of a busy script. Signed-off-by: zyxwvu Shi (cherry picked from commit f61c37cec900ba391541f20f7655aad44a26bafc) --- src/db.c | 2 +- src/scripting.c | 11 +++++++---- src/server.h | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/db.c b/src/db.c index 40377ec3f..840e95e21 100644 --- a/src/db.c +++ b/src/db.c @@ -1480,7 +1480,7 @@ int keyIsExpired(redisDb *db, robj *key) { * script execution, making propagation to slaves / AOF consistent. * See issue #1525 on Github for more information. */ if (server.lua_caller) { - now = server.lua_time_start; + now = server.lua_time_snapshot; } /* If we are in the middle of a command execution, we still want to use * a reference time that does not change: in that case we just use the diff --git a/src/scripting.c b/src/scripting.c index 299e60810..dbbd50eaf 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -31,6 +31,7 @@ #include "sha1.h" #include "rand.h" #include "cluster.h" +#include "monotonic.h" #include #include @@ -1427,7 +1428,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) { /* This is the Lua script "count" hook that we use to detect scripts timeout. */ void luaMaskCountHook(lua_State *lua, lua_Debug *ar) { - long long elapsed = mstime() - server.lua_time_start; + long long elapsed = elapsedMs(server.lua_time_start); UNUSED(ar); UNUSED(lua); @@ -1578,7 +1579,8 @@ void evalGenericCommand(client *c, int evalsha) { server.in_eval = 1; server.lua_caller = c; server.lua_cur_script = funcname + 2; - server.lua_time_start = mstime(); + server.lua_time_start = getMonotonicUs(); + server.lua_time_snapshot = mstime(); server.lua_kill = 0; if (server.lua_time_limit > 0 && ldb.active == 0) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); @@ -2729,7 +2731,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { /* Check if a timeout occurred. */ if (ar->event == LUA_HOOKCOUNT && ldb.step == 0 && bp == 0) { - mstime_t elapsed = mstime() - server.lua_time_start; + mstime_t elapsed = elapsedMs(server.lua_time_start); mstime_t timelimit = server.lua_time_limit ? server.lua_time_limit : 5000; if (elapsed >= timelimit) { @@ -2759,6 +2761,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) { lua_pushstring(lua, "timeout during Lua debugging with client closing connection"); lua_error(lua); } - server.lua_time_start = mstime(); + server.lua_time_start = getMonotonicUs(); + server.lua_time_snapshot = mstime(); } } diff --git a/src/server.h b/src/server.h index a2a722c6d..9b7e16a0d 100644 --- a/src/server.h +++ b/src/server.h @@ -1571,7 +1571,8 @@ struct redisServer { dict *lua_scripts; /* A dictionary of SHA1 -> Lua scripts */ unsigned long long lua_scripts_mem; /* Cached scripts' memory + oh */ mstime_t lua_time_limit; /* Script timeout in milliseconds */ - mstime_t lua_time_start; /* Start time of script, milliseconds time */ + monotime lua_time_start; /* monotonic timer to detect timed-out script */ + mstime_t lua_time_snapshot; /* Snapshot of mstime when script is started */ int lua_write_dirty; /* True if a write command was called during the execution of the current script. */ int lua_random_dirty; /* True if a random command was called during the From 9158a510e1f06e8460447e0210deed9dc7dd03f3 Mon Sep 17 00:00:00 2001 From: Istemi Ekin Akkus <5419814+iakkus@users.noreply.github.com> Date: Sun, 25 Apr 2021 09:05:12 +0200 Subject: [PATCH 06/24] Modules: Fix RM_GetClusterNodeInfo() to correctly populate the master_id (#8846) (cherry picked from commit af035c1e1d3bcaf662051cff4dc49f6051321c9c) --- src/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index 05bf3a275..727cdc43f 100644 --- a/src/module.c +++ b/src/module.c @@ -6168,7 +6168,7 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m /* If the information is not available, the function will set the * field to zero bytes, so that when the field can't be populated the * function kinda remains predictable. */ - if (node->flags & CLUSTER_NODE_MASTER && node->slaveof) + if (node->flags & CLUSTER_NODE_SLAVE && node->slaveof) memcpy(master_id,node->slaveof->name,REDISMODULE_NODE_ID_LEN); else memset(master_id,0,REDISMODULE_NODE_ID_LEN); From ce288cb5598e6735e2fc84bda9d89b41111249c0 Mon Sep 17 00:00:00 2001 From: Yang Bodong Date: Sun, 25 Apr 2021 19:00:35 +0800 Subject: [PATCH 07/24] When the password is wrong, redis-benchmark should exit (#8855) (cherry picked from commit 8423b77f14c0d3a58e580c65a70b4f980f5cdcf6) --- src/redis-benchmark.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 351335862..068c094e1 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -1782,8 +1782,10 @@ int main(int argc, const char **argv) { } else { config.redis_config = getRedisConfig(config.hostip, config.hostport, config.hostsocket); - if (config.redis_config == NULL) + if (config.redis_config == NULL) { fprintf(stderr, "WARN: could not fetch server CONFIG\n"); + exit(1); + } } if (config.num_threads > 0) { pthread_mutex_init(&(config.liveclients_mutex), NULL); From 9d71ac0a6ea2586a3a7e452cd4449b62993c4309 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Mon, 26 Apr 2021 18:43:57 +0300 Subject: [PATCH 08/24] Remove redundant -latomic on arm64. (#8867) (cherry picked from commit ebfbb091096b6f36cf82e9f6e6583b10fd5b5acb) --- src/Makefile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Makefile b/src/Makefile index 28d50da02..62e37cb48 100644 --- a/src/Makefile +++ b/src/Makefile @@ -93,14 +93,10 @@ FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) FINAL_LIBS=-lm DEBUG=-g -ggdb -# Linux ARM needs -latomic at linking time -ifneq (,$(filter aarch64 armv,$(uname_M))) - FINAL_LIBS+=-latomic -else +# Linux ARM32 needs -latomic at linking time ifneq (,$(findstring armv,$(uname_M))) FINAL_LIBS+=-latomic endif -endif ifeq ($(uname_S),SunOS) # SunOS From 91c450e80a336f8966da713d2ba137b6e9c3acfd Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 27 Apr 2021 08:15:10 +0300 Subject: [PATCH 09/24] Prevent replicas from sending commands that interact with keyspace (#8868) This solves an issue reported in #8712 in which a replica would bypass the client write pause check and cause an assertion due to executing a write command during failover. The fact is that we don't expect replicas to execute any command other than maybe REPLCONF and PING, etc. but matching against the ADMIN command flag is insufficient, so instead i just block keyspace access for now. (cherry picked from commit 46f4ebbe842620f0976a36741a72482620aa4b48) --- src/server.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 993260619..1fe150b3b 100644 --- a/src/server.c +++ b/src/server.c @@ -3985,6 +3985,8 @@ int processCommand(client *c) { return C_OK; } + int is_read_command = (c->cmd->flags & CMD_READONLY) || + (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_READONLY)); int is_write_command = (c->cmd->flags & CMD_WRITE) || (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE)); int is_denyoom_command = (c->cmd->flags & CMD_DENYOOM) || @@ -4194,7 +4196,7 @@ int processCommand(client *c) { c->cmd->proc != discardCommand && c->cmd->proc != watchCommand && c->cmd->proc != unwatchCommand && - c->cmd->proc != resetCommand && + c->cmd->proc != resetCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && @@ -4206,6 +4208,14 @@ int processCommand(client *c) { return C_OK; } + /* Prevent a replica from sending commands that access the keyspace. + * The main objective here is to prevent abuse of client pause check + * from which replicas are exempt. */ + if ((c->flags & CLIENT_SLAVE) && (is_may_replicate_command || is_write_command || is_read_command)) { + rejectCommandFormat(c, "Replica can't interract with the keyspace"); + return C_OK; + } + /* If the server is paused, block the client until * the pause has ended. Replicas are never paused. */ if (!(c->flags & CLIENT_SLAVE) && From 2149c5a03041992104b6a8a5847b6d630319bb06 Mon Sep 17 00:00:00 2001 From: yoav-steinberg Date: Tue, 27 Apr 2021 16:22:22 +0300 Subject: [PATCH 10/24] Bump freebsd-vm version to fix CI failures (#8876) Specifically we had issues with NTP sync failure which was resolved here: https://github.com/vmactions/freebsd-vm/commit/457af7345642e154a79d219971a2d4a7c7fe2118 (cherry picked from commit 2e88b0639689a3019e27f55dfa40578847443eeb) --- .github/workflows/daily.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index ee9ac1bbf..9e4630e29 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -253,7 +253,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: test - uses: vmactions/freebsd-vm@v0.1.2 + uses: vmactions/freebsd-vm@v0.1.4 with: usesh: true sync: rsync From 9868ad801d7f3aa608d7331fd51f4cc3ef0f4100 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Tue, 27 Apr 2021 23:02:23 +0800 Subject: [PATCH 11/24] Fix potential CONFIG SET bind test failure. (#8875) Use an invalid IP address to trigger CONFIG SET bind failure, instead of DNS which is not guaranteed to always fail. (cherry picked from commit 2b22fffc787e91df789dabf23ddcf19ecf34cf6f) --- tests/unit/networking.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/networking.tcl b/tests/unit/networking.tcl index 19feee8c3..38b49d45e 100644 --- a/tests/unit/networking.tcl +++ b/tests/unit/networking.tcl @@ -25,7 +25,7 @@ test {CONFIG SET port number} { test {CONFIG SET bind address} { start_server {} { # non-valid address - catch {r CONFIG SET bind "some.wrong.bind.address"} e + catch {r CONFIG SET bind "999.999.999.999"} e assert_match {*Failed to bind to specified addresses*} $e # make sure server still bound to the previous address @@ -33,4 +33,4 @@ test {CONFIG SET bind address} { $rd PING $rd close } -} \ No newline at end of file +} From ee542fbbd9065ae1c527bbc4bf77c3eba3a77cc1 Mon Sep 17 00:00:00 2001 From: filipe oliveira Date: Wed, 28 Apr 2021 07:51:07 +0100 Subject: [PATCH 12/24] redis-benchmark: Error/Warning handling updates. (#8869) - Immediately exit on errors that are not related to topology updates. - Deprecates the `-e` option ( retro compatible ) and warns that we now exit immediately on errors that are not related to topology updates. - Fixed wrongfully failing on config fetch error (warning only). This only affects RE. Bottom line: - MOVED and ASK errors will not show any warning (unlike the throttled error with `-e` before). - CLUSTERDOWN still prints an error unconditionally and sleeps for 1 second. - other errors are fatal. (cherry picked from commit ef6f902372d4646b1894ec5dbd5f857dea5688d6) --- src/redis-benchmark.c | 82 +++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 068c094e1..51dba9511 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -99,7 +99,6 @@ static struct config { int randomkeys_keyspacelen; int keepalive; int pipeline; - int showerrors; long long start; long long totlatency; const char *title; @@ -307,7 +306,9 @@ static redisContext *getRedisContext(const char *ip, int port, fprintf(stderr, "Node %s:%d replied with error:\n%s\n", ip, port, reply->str); else fprintf(stderr, "Node %s replied with error:\n%s\n", hostsocket, reply->str); - goto cleanup; + freeReplyObject(reply); + redisFree(ctx); + exit(1); } freeReplyObject(reply); return ctx; @@ -366,9 +367,15 @@ fail: fprintf(stderr, "ERROR: failed to fetch CONFIG from "); if (hostsocket == NULL) fprintf(stderr, "%s:%d\n", ip, port); else fprintf(stderr, "%s\n", hostsocket); + int abort_test = 0; + if (!strncmp(reply->str,"NOAUTH",5) || + !strncmp(reply->str,"WRONGPASS",9) || + !strncmp(reply->str,"NOPERM",5)) + abort_test = 1; freeReplyObject(reply); redisFree(c); freeRedisConfig(cfg); + if (abort_test) exit(1); return NULL; } static void freeRedisConfig(redisConfig *cfg) { @@ -513,44 +520,39 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) { exit(1); } redisReply *r = reply; - int is_err = (r->type == REDIS_REPLY_ERROR); - - if (is_err && config.showerrors) { - /* TODO: static lasterr_time not thread-safe */ - static time_t lasterr_time = 0; - time_t now = time(NULL); - if (lasterr_time != now) { - lasterr_time = now; - if (c->cluster_node) { - printf("Error from server %s:%d: %s\n", + if (r->type == REDIS_REPLY_ERROR) { + /* Try to update slots configuration if reply error is + * MOVED/ASK/CLUSTERDOWN and the key(s) used by the command + * contain(s) the slot hash tag. + * If the error is not topology-update related then we + * immediately exit to avoid false results. */ + if (c->cluster_node && c->staglen) { + int fetch_slots = 0, do_wait = 0; + if (!strncmp(r->str,"MOVED",5) || !strncmp(r->str,"ASK",3)) + fetch_slots = 1; + else if (!strncmp(r->str,"CLUSTERDOWN",11)) { + /* Usually the cluster is able to recover itself after + * a CLUSTERDOWN error, so try to sleep one second + * before requesting the new configuration. */ + fetch_slots = 1; + do_wait = 1; + printf("Error from server %s:%d: %s.\n", c->cluster_node->ip, c->cluster_node->port, r->str); + } + if (do_wait) sleep(1); + if (fetch_slots && !fetchClusterSlotsConfiguration(c)) + exit(1); + } else { + if (c->cluster_node) { + printf("Error from server %s:%d: %s\n", + c->cluster_node->ip, + c->cluster_node->port, + r->str); } else printf("Error from server: %s\n", r->str); - } - } - - /* Try to update slots configuration if reply error is - * MOVED/ASK/CLUSTERDOWN and the key(s) used by the command - * contain(s) the slot hash tag. */ - if (is_err && c->cluster_node && c->staglen) { - int fetch_slots = 0, do_wait = 0; - if (!strncmp(r->str,"MOVED",5) || !strncmp(r->str,"ASK",3)) - fetch_slots = 1; - else if (!strncmp(r->str,"CLUSTERDOWN",11)) { - /* Usually the cluster is able to recover itself after - * a CLUSTERDOWN error, so try to sleep one second - * before requesting the new configuration. */ - fetch_slots = 1; - do_wait = 1; - printf("Error from server %s:%d: %s\n", - c->cluster_node->ip, - c->cluster_node->port, - r->str); - } - if (do_wait) sleep(1); - if (fetch_slots && !fetchClusterSlotsConfiguration(c)) exit(1); + } } freeReplyObject(reply); @@ -1293,8 +1295,7 @@ static int fetchClusterSlotsConfiguration(client c) { atomicGetIncr(config.is_fetching_slots, is_fetching_slots, 1); if (is_fetching_slots) return -1; //TODO: use other codes || errno ? atomicSet(config.is_fetching_slots, 1); - if (config.showerrors) - printf("Cluster slots configuration changed, fetching new one...\n"); + printf("WARNING: Cluster slots configuration changed, fetching new one...\n"); const char *errmsg = "Failed to update cluster slots configuration"; static dictType dtype = { dictSdsHash, /* hash function */ @@ -1470,7 +1471,8 @@ int parseOptions(int argc, const char **argv) { } else if (!strcmp(argv[i],"-I")) { config.idlemode = 1; } else if (!strcmp(argv[i],"-e")) { - config.showerrors = 1; + printf("WARNING: -e option has been deprecated. " + "We now immediatly exit on error to avoid false results.\n"); } else if (!strcmp(argv[i],"-t")) { if (lastarg) goto invalid; /* We get the list of tests to run as a string in the form @@ -1573,8 +1575,6 @@ usage: " is executed. Default tests use this to hit random keys in the\n" " specified range.\n" " -P Pipeline requests. Default 1 (no pipeline).\n" -" -e If server replies with errors, show them on stdout.\n" -" (no more than 1 error per second is displayed)\n" " -q Quiet. Just show query/sec values\n" " --precision Number of decimal places to display in latency output (default 0)\n" " --csv Output in CSV format\n" @@ -1699,7 +1699,6 @@ int main(int argc, const char **argv) { config.keepalive = 1; config.datasize = 3; config.pipeline = 1; - config.showerrors = 0; config.randomkeys = 0; config.randomkeys_keyspacelen = 0; config.quiet = 0; @@ -1784,7 +1783,6 @@ int main(int argc, const char **argv) { getRedisConfig(config.hostip, config.hostport, config.hostsocket); if (config.redis_config == NULL) { fprintf(stderr, "WARN: could not fetch server CONFIG\n"); - exit(1); } } if (config.num_threads > 0) { From e919deac128d0648407bc163d639527e1ca7a031 Mon Sep 17 00:00:00 2001 From: Binbin Date: Wed, 28 Apr 2021 18:19:55 +0800 Subject: [PATCH 13/24] redis-cli: Do not use hostsocket when we got redirected in cluster mode (#8870) When redis-cli was used with both -c (cluster) and -s (unix socket), it would have kept trying to use that unix socket, even if it got redirected by the cluster (resulting in an infinite loop). (cherry picked from commit 416f2773395ffcd72d8d8408e1558f49d59a0077) --- src/redis-cli.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 7e1fe3934..ff34f2b6a 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -844,7 +844,9 @@ static int cliConnect(int flags) { cliRefreshPrompt(); } - if (config.hostsocket == NULL) { + /* Do not use hostsocket when we got redirected in cluster mode */ + if (config.hostsocket == NULL || + (config.cluster_mode && config.cluster_reissue_command)) { context = redisConnect(config.hostip,config.hostport); } else { context = redisConnectUnix(config.hostsocket); From 0d10460235fa1d13ef4574e3af7a2288ca1ec08e Mon Sep 17 00:00:00 2001 From: Binbin Date: Wed, 28 Apr 2021 21:03:24 +0800 Subject: [PATCH 14/24] redis-benchmark: Add zfree(data) and fix lrange size / text mismatch (#8872) missing zfree(data) in redis-benchmark. And also correct the wrong size in lrange. the text mentioned 500, but size was 450, changed to 500 (cherry picked from commit 1eff8564c78011f7257e485796990a0d4d607a5b) --- src/redis-benchmark.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 51dba9511..fa024d44f 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -1946,8 +1946,8 @@ int main(int argc, const char **argv) { } if (test_is_selected("lrange") || test_is_selected("lrange_500")) { - len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 449",tag); - benchmark("LRANGE_500 (first 450 elements)",cmd,len); + len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 499",tag); + benchmark("LRANGE_500 (first 500 elements)",cmd,len); free(cmd); } @@ -1974,6 +1974,7 @@ int main(int argc, const char **argv) { if (!config.csv) printf("\n"); } while(config.loop); + zfree(data); if (config.redis_config != NULL) freeRedisConfig(config.redis_config); return 0; From 02bd008bd1a8ba62811b794d71741a6b9af7baeb Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Thu, 29 Apr 2021 17:08:52 +0800 Subject: [PATCH 15/24] Improve redis-cli help. When help command, we only match command (#8879) prefix args not all args. So when we help commands with subcommands, all subcommands will be output. (cherry picked from commit 0b1b9edb2843730b03f78b6073cdd30873dbba95) --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index ff34f2b6a..9f88a9c88 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -663,7 +663,7 @@ static void cliOutputHelp(int argc, char **argv) { help = entry->org; if (group == -1) { /* Compare all arguments */ - if (argc == entry->argc) { + if (argc <= entry->argc) { for (j = 0; j < argc; j++) { if (strcasecmp(argv[j],entry->argv[j]) != 0) break; } From 23e126a9933c0cfe8cb3cdced7fb7efd67acb45f Mon Sep 17 00:00:00 2001 From: sundb Date: Sun, 2 May 2021 15:32:57 +0800 Subject: [PATCH 16/24] Fix memory leak in moduleDefragGlobals (#8853) (cherry picked from commit 5100ef9f8246dec6590f35f6b9f0b88c2dea0cfb) --- src/module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.c b/src/module.c index 727cdc43f..c76241bfb 100644 --- a/src/module.c +++ b/src/module.c @@ -9205,6 +9205,7 @@ long moduleDefragGlobals(void) { module->defrag_cb(&defrag_ctx); defragged += defrag_ctx.defragged; } + dictReleaseIterator(di); return defragged; } From 832b7682480f5057cab4b314a5f73b6073f963a7 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 May 2021 08:27:22 +0300 Subject: [PATCH 17/24] Fix integer overflow in intset (CVE-2021-29478) An integer overflow bug in Redis 6.2 could be exploited to corrupt the heap and potentially result with remote code execution. The vulnerability involves changing the default set-max-intset-entries configuration value, creating a large set key that consists of integer values and using the COPY command to duplicate it. The integer overflow bug exists in all versions of Redis starting with 2.6, where it could result with a corrupted RDB or DUMP payload, but not exploited through COPY (which did not exist before 6.2). (cherry picked from commit 29900d4e6bccdf3691bedf0ea9a5d84863fa3592) --- src/intset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intset.c b/src/intset.c index 1a64ecae8..9ba13898d 100644 --- a/src/intset.c +++ b/src/intset.c @@ -281,7 +281,7 @@ uint32_t intsetLen(const intset *is) { /* Return intset blob size in bytes. */ size_t intsetBlobLen(intset *is) { - return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding); + return sizeof(intset)+(size_t)intrev32ifbe(is->length)*intrev32ifbe(is->encoding); } /* Validate the integrity of the data structure. From fe0345931c93f5a46f35f0d92c28f5347f2062f2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 May 2021 08:32:31 +0300 Subject: [PATCH 18/24] Fix integer overflow in STRALGO LCS (CVE-2021-29477) An integer overflow bug in Redis version 6.0 or newer could be exploited using the STRALGO LCS command to corrupt the heap and potentially result with remote code execution. (cherry picked from commit f0c5f920d0f88bd8aa376a2c05af4902789d1ef9) --- src/t_string.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index 0967e30e1..490d5983a 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -805,7 +805,7 @@ void stralgoLCS(client *c) { /* Setup an uint32_t array to store at LCS[i,j] the length of the * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so * we index it as LCS[j+(blen+1)*j] */ - uint32_t *lcs = zmalloc((alen+1)*(blen+1)*sizeof(uint32_t)); + uint32_t *lcs = zmalloc((size_t)(alen+1)*(blen+1)*sizeof(uint32_t)); #define LCS(A,B) lcs[(B)+((A)*(blen+1))] /* Start building the LCS table. */ From f72cad07e1d9b5710833c89d737623b7ccfc83d2 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 May 2021 08:33:21 +0300 Subject: [PATCH 19/24] Resolve nonsense static analysis warnings (cherry picked from commit fd7d51c353607f350c865155444bce9236f3d682) --- .../jemalloc/internal/jemalloc_internal_inlines_c.h | 2 +- src/lolwut.c | 4 ++-- src/memtest.c | 2 +- src/object.c | 2 +- src/redis-check-rdb.c | 2 +- src/redis-cli.c | 6 +++--- src/sentinel.c | 12 ++++++------ 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 2685802b8..b19a94207 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -235,7 +235,7 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr) { int free_in_slab = extent_nfree_get(slab); if (free_in_slab) { const bin_info_t *bin_info = &bin_infos[binind]; - int curslabs = bin->stats.curslabs; + unsigned long curslabs = bin->stats.curslabs; size_t curregs = bin->stats.curregs; if (bin->slabcur) { /* remove slabcur from the overall utilization */ diff --git a/src/lolwut.c b/src/lolwut.c index eebd5da6a..931f311cd 100644 --- a/src/lolwut.c +++ b/src/lolwut.c @@ -94,8 +94,8 @@ lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) { lwCanvas *canvas = zmalloc(sizeof(*canvas)); canvas->width = width; canvas->height = height; - canvas->pixels = zmalloc(width*height); - memset(canvas->pixels,bgcolor,width*height); + canvas->pixels = zmalloc((size_t)width*height); + memset(canvas->pixels,bgcolor,(size_t)width*height); return canvas; } diff --git a/src/memtest.c b/src/memtest.c index cb4d35e83..bc0ac3a66 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -71,7 +71,7 @@ void memtest_progress_start(char *title, int pass) { printf("\x1b[H\x1b[2K"); /* Cursor home, clear current line. */ printf("%s [%d]\n", title, pass); /* Print title. */ progress_printed = 0; - progress_full = ws.ws_col*(ws.ws_row-3); + progress_full = (size_t)ws.ws_col*(ws.ws_row-3); fflush(stdout); } diff --git a/src/object.c b/src/object.c index b75e547b9..c7b25ffd4 100644 --- a/src/object.c +++ b/src/object.c @@ -836,7 +836,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) { if (samples) asize += (double)elesize/samples*dictSize(d); } else if (o->encoding == OBJ_ENCODING_INTSET) { intset *is = o->ptr; - asize = sizeof(*o)+sizeof(*is)+is->encoding*is->length; + asize = sizeof(*o)+sizeof(*is)+(size_t)is->encoding*is->length; } else { serverPanic("Unknown set encoding"); } diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 4f451969a..6ddfda7ff 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -250,7 +250,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { rdbstate.doing = RDB_CHECK_DOING_READ_LEN; if ((dbid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR) goto eoferr; - rdbCheckInfo("Selecting DB ID %d", dbid); + rdbCheckInfo("Selecting DB ID %llu", (unsigned long long)dbid); continue; /* Read type again. */ } else if (type == RDB_OPCODE_RESIZEDB) { /* RESIZEDB: Hint about the size of the keys in the currently diff --git a/src/redis-cli.c b/src/redis-cli.c index 9f88a9c88..81be58b17 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -5483,7 +5483,7 @@ static void clusterManagerNodeArrayReset(clusterManagerNodeArray *array) { static void clusterManagerNodeArrayShift(clusterManagerNodeArray *array, clusterManagerNode **nodeptr) { - assert(array->nodes < (array->nodes + array->len)); + assert(array->len > 0); /* If the first node to be shifted is not NULL, decrement count. */ if (*array->nodes != NULL) array->count--; /* Store the first node to be shifted into 'nodeptr'. */ @@ -5496,7 +5496,7 @@ static void clusterManagerNodeArrayShift(clusterManagerNodeArray *array, static void clusterManagerNodeArrayAdd(clusterManagerNodeArray *array, clusterManagerNode *node) { - assert(array->nodes < (array->nodes + array->len)); + assert(array->len > 0); assert(node != NULL); assert(array->count < array->len); array->nodes[array->count++] = node; @@ -6873,7 +6873,7 @@ void showLatencyDistSamples(struct distsamples *samples, long long tot) { printf("\033[38;5;0m"); /* Set foreground color to black. */ for (j = 0; ; j++) { int coloridx = - ceil((float) samples[j].count / tot * (spectrum_palette_size-1)); + ceil((double) samples[j].count / tot * (spectrum_palette_size-1)); int color = spectrum_palette[coloridx]; printf("\033[48;5;%dm%c", (int)color, samples[j].character); samples[j].count = 0; diff --git a/src/sentinel.c b/src/sentinel.c index a56cd8b15..2d81d98ac 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -4119,16 +4119,16 @@ void sentinelSetCommand(client *c) { int numargs = j-old_j+1; switch(numargs) { case 2: - sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s",c->argv[old_j]->ptr, - c->argv[old_j+1]->ptr); + sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s",(char*)c->argv[old_j]->ptr, + (char*)c->argv[old_j+1]->ptr); break; case 3: - sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s %s",c->argv[old_j]->ptr, - c->argv[old_j+1]->ptr, - c->argv[old_j+2]->ptr); + sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s %s",(char*)c->argv[old_j]->ptr, + (char*)c->argv[old_j+1]->ptr, + (char*)c->argv[old_j+2]->ptr); break; default: - sentinelEvent(LL_WARNING,"+set",ri,"%@ %s",c->argv[old_j]->ptr); + sentinelEvent(LL_WARNING,"+set",ri,"%@ %s",(char*)c->argv[old_j]->ptr); break; } } From 439c356fe600b0ce29728ddca8961eb4c09414a9 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 3 May 2021 12:08:20 +0300 Subject: [PATCH 20/24] Redis 6.2.3 --- 00-RELEASENOTES | 34 ++++++++++++++++++++++++++++++++++ src/version.h | 4 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 8a1405e41..4f6cb9978 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,40 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.2.3 Released Mon May 3 19:00:00 IST 2021 +================================================================================ + +Upgrade urgency: SECURITY, Contains fixes to security issues that affect +authenticated client connections. LOW otherwise. + +Integer overflow in STRALGO LCS command (CVE-2021-29477): +An integer overflow bug in Redis version 6.0 or newer could be exploited using +the STRALGO LCS command to corrupt the heap and potentially result in remote +code execution. The integer overflow bug exists in all versions of Redis +starting with 6.0. + +Integer overflow in COPY command for large intsets (CVE-2021-29478): +An integer overflow bug in Redis 6.2 could be exploited to corrupt the heap and +potentially result with remote code execution. The vulnerability involves +changing the default set-max-intset-entries configuration value, creating a +large set key that consists of integer values and using the COPY command to +duplicate it. The integer overflow bug exists in all versions of Redis starting +with 2.6, where it could result with a corrupted RDB or DUMP payload, but not +exploited through COPY (which did not exist before 6.2). + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix memory leak in moduleDefragGlobals (#8853) +* Fix memory leak when doing lazy freeing client tracking table (#8822) +* Block abusive replicas from sending command that could assert and crash redis (#8868) + +Other bug fixes: +* Use a monotonic clock to check for Lua script timeout (#8812) +* redis-cli: Do not use unix socket when we got redirected in cluster mode (#8870) + +Modules: +* Fix RM_GetClusterNodeInfo() to correctly populate master id (#8846) + ================================================================================ Redis 6.2.2 Released Mon April 19 19:00:00 IST 2021 ================================================================================ diff --git a/src/version.h b/src/version.h index 3c5dc02c5..b87f2b9c3 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.2.2" -#define REDIS_VERSION_NUM 0x00060202 +#define REDIS_VERSION "6.2.3" +#define REDIS_VERSION_NUM 0x00060203 From a9552f9635216699d7f64aed8c969a9f93d4e500 Mon Sep 17 00:00:00 2001 From: christianEQ Date: Thu, 20 May 2021 19:09:02 +0000 Subject: [PATCH 21/24] license status OK if not checking license Former-commit-id: 6bfdc9d41dc638989d50f005af8d66e4ed47ce77 --- src/server.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server.cpp b/src/server.cpp index bc268b4fd..cbb7654e1 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5474,7 +5474,11 @@ sds genRedisInfoString(const char *section) { "variant:enterprise\r\n" "license_status:%s\r\n" "mvcc_depth:%d\r\n", + #ifdef NO_LICENSE_CHECK + "OK", + #else cserver.license_key ? "OK" : "Trial", + #endif mvcc_depth ); } From 982067b16a3162d2c26dcd11a4d5db662dacaf47 Mon Sep 17 00:00:00 2001 From: christianEQ Date: Wed, 2 Jun 2021 15:02:27 +0000 Subject: [PATCH 22/24] fixed code style for ifdef Former-commit-id: 93c41fa31c91098af98d2bc0362eb29685cd6678 --- src/server.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.cpp b/src/server.cpp index cbb7654e1..1424ffbd1 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5474,11 +5474,11 @@ sds genRedisInfoString(const char *section) { "variant:enterprise\r\n" "license_status:%s\r\n" "mvcc_depth:%d\r\n", - #ifdef NO_LICENSE_CHECK +#ifdef NO_LICENSE_CHECK "OK", - #else +#else cserver.license_key ? "OK" : "Trial", - #endif +#endif mvcc_depth ); } From dd8e8b098c954b4ea19c92ca631fef81168c7065 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 14 Jun 2021 16:32:47 +0000 Subject: [PATCH 23/24] active defrag tests need to run single threaded because jemalloc has seperate mempools per thread and the numbers won't match otherwise Former-commit-id: 3a1d3090f2ec5a442e3a7c192987cdfa24094145 --- tests/unit/memefficiency.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index d5c2feb4f..2a2db72cd 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -395,7 +395,7 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent # if the current slab is lower in utilization the defragger would have ended up in stagnation, # keept running and not move any allocation. # this test is more consistent on a fresh server with no history - start_server {tags {"defrag"} overrides {save ""}} { + start_server {tags {"defrag"} overrides {save "" server-threads 1}} { r flushdb r config resetstat r config set hz 100 From 420b07960c358ff02cbd54a1f4795a0972f8f036 Mon Sep 17 00:00:00 2001 From: John Sully Date: Mon, 14 Jun 2021 22:06:36 +0000 Subject: [PATCH 24/24] Prevent test code crash due to no log data Former-commit-id: 0a56a73bd98d4e692ae77683fdb9dd644ecfc2eb --- tests/integration/psync2.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/psync2.tcl b/tests/integration/psync2.tcl index 8459d2378..eccf6df2d 100644 --- a/tests/integration/psync2.tcl +++ b/tests/integration/psync2.tcl @@ -39,6 +39,7 @@ proc show_cluster_status {} { # all the lists are empty. # # regexp {^[0-9]+:[A-Z] [0-9]+ [A-z]+ [0-9]+ ([0-9:.]+) .*} $l - logdate + catch { while 1 { # Find the log with smallest time. set empty 0 @@ -67,6 +68,7 @@ proc show_cluster_status {} { puts "\[$best port $R_port($best)\] [lindex $log($best) 0]" set log($best) [lrange $log($best) 1 end] } + } } }