From 7059cfb432e33bfffcd80c3443cc3925f27e33f4 Mon Sep 17 00:00:00 2001 From: yoav-steinberg Date: Tue, 4 May 2021 13:45:08 +0300 Subject: [PATCH 01/79] Enforce client output buffer soft limit when no traffic. (#8833) When client breached the output buffer soft limit but then went idle, we didn't disconnect on soft limit timeout, now we do. Note this also resolves some sporadic test failures in due to Linux buffering data which caused tests to fail if during the test we went back under the soft COB limit. Co-authored-by: Oran Agra Co-authored-by: sundb (cherry picked from commit 152fce5e2cbf947a389da414a431f7331981a374) --- src/networking.c | 33 ++++++++--- src/server.c | 1 + src/server.h | 2 +- tests/unit/obuf-limits.tcl | 111 ++++++++++++++++++++----------------- 4 files changed, 85 insertions(+), 62 deletions(-) diff --git a/src/networking.c b/src/networking.c index 2355a376b..14e94b84c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -333,7 +333,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) { listAddNodeTail(c->reply, tail); c->reply_bytes += tail->size; - asyncCloseClientOnOutputBufferLimitReached(c); + closeClientOnOutputBufferLimitReached(c, 1); } } @@ -616,7 +616,7 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { listNodeValue(ln) = buf; c->reply_bytes += buf->size; - asyncCloseClientOnOutputBufferLimitReached(c); + closeClientOnOutputBufferLimitReached(c, 1); } } @@ -949,7 +949,7 @@ void AddReplyFromClient(client *dst, client *src) { src->bufpos = 0; /* Check output buffer limits */ - asyncCloseClientOnOutputBufferLimitReached(dst); + closeClientOnOutputBufferLimitReached(dst, 1); } /* Copy 'src' client output buffers into 'dst' client output buffers. @@ -3223,18 +3223,33 @@ int checkClientOutputBufferLimits(client *c) { * * Note: we need to close the client asynchronously because this function is * called from contexts where the client can't be freed safely, i.e. from the - * lower level functions pushing data inside the client output buffers. */ -void asyncCloseClientOnOutputBufferLimitReached(client *c) { - if (!c->conn) return; /* It is unsafe to free fake clients. */ + * lower level functions pushing data inside the client output buffers. + * When `async` is set to 0, we close the client immediately, this is + * useful when called from cron. + * + * Returns 1 if client was (flagged) closed. */ +int closeClientOnOutputBufferLimitReached(client *c, int async) { + if (!c->conn) return 0; /* It is unsafe to free fake clients. */ serverAssert(c->reply_bytes < SIZE_MAX-(1024*64)); - if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return; + if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return 0; if (checkClientOutputBufferLimits(c)) { sds client = catClientInfoString(sdsempty(),c); - freeClientAsync(c); - serverLog(LL_WARNING,"Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", client); + if (async) { + freeClientAsync(c); + serverLog(LL_WARNING, + "Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", + client); + } else { + freeClient(c); + serverLog(LL_WARNING, + "Client %s closed for overcoming of output buffer limits.", + client); + } sdsfree(client); + return 1; } + return 0; } /* Helper function used by performEvictions() in order to flush slaves diff --git a/src/server.c b/src/server.c index 1fe150b3b..9541aada7 100644 --- a/src/server.c +++ b/src/server.c @@ -1839,6 +1839,7 @@ void clientsCron(void) { if (clientsCronResizeQueryBuffer(c)) continue; if (clientsCronTrackExpansiveClients(c, curr_peak_mem_usage_slot)) continue; if (clientsCronTrackClientsMemUsage(c)) continue; + if (closeClientOnOutputBufferLimitReached(c, 0)) continue; } } diff --git a/src/server.h b/src/server.h index 9b7e16a0d..ee87ecacf 100644 --- a/src/server.h +++ b/src/server.h @@ -1867,7 +1867,7 @@ void rewriteClientCommandArgument(client *c, int i, robj *newval); void replaceClientCommandVector(client *c, int argc, robj **argv); unsigned long getClientOutputBufferMemoryUsage(client *c); int freeClientsInAsyncFreeQueue(void); -void asyncCloseClientOnOutputBufferLimitReached(client *c); +int closeClientOnOutputBufferLimitReached(client *c, int async); int getClientType(client *c); int getClientTypeByName(char *name); char *getClientTypeName(int class); diff --git a/tests/unit/obuf-limits.tcl b/tests/unit/obuf-limits.tcl index 38a643385..3f26c53f9 100644 --- a/tests/unit/obuf-limits.tcl +++ b/tests/unit/obuf-limits.tcl @@ -18,65 +18,72 @@ start_server {tags {"obuf-limits"}} { assert {$omem >= 70000 && $omem < 200000} $rd1 close } - - test {Client output buffer soft limit is not enforced if time is not overreached} { - r config set client-output-buffer-limit {pubsub 0 100000 10} - set rd1 [redis_deferring_client] - - $rd1 subscribe foo - set reply [$rd1 read] - assert {$reply eq "subscribe foo 1"} - - set omem 0 - set start_time 0 - set time_elapsed 0 - while 1 { - if {$start_time != 0} { - # Slow down loop when omen has reached the limit. - after 10 - } - r publish foo [string repeat "x" 1000] - set clients [split [r client list] "\r\n"] - set c [split [lindex $clients 1] " "] - if {![regexp {omem=([0-9]+)} $c - omem]} break - if {$omem > 100000} { - if {$start_time == 0} {set start_time [clock seconds]} - set time_elapsed [expr {[clock seconds]-$start_time}] - if {$time_elapsed >= 5} break - } + + foreach {soft_limit_time wait_for_timeout} {3 yes + 4 no } { + if $wait_for_timeout { + set test_name "Client output buffer soft limit is enforced if time is overreached" + } else { + set test_name "Client output buffer soft limit is not enforced too early and is enforced when no traffic" } - assert {$omem >= 100000 && $time_elapsed >= 5 && $time_elapsed <= 10} - $rd1 close - } - test {Client output buffer soft limit is enforced if time is overreached} { - r config set client-output-buffer-limit {pubsub 0 100000 3} - set rd1 [redis_deferring_client] + test $test_name { + r config set client-output-buffer-limit "pubsub 0 100000 $soft_limit_time" + set soft_limit_time [expr $soft_limit_time*1000] + set rd1 [redis_deferring_client] - $rd1 subscribe foo - set reply [$rd1 read] - assert {$reply eq "subscribe foo 1"} + $rd1 client setname test_client + set reply [$rd1 read] + assert {$reply eq "OK"} - set omem 0 - set start_time 0 - set time_elapsed 0 - while 1 { - if {$start_time != 0} { - # Slow down loop when omen has reached the limit. - after 10 + $rd1 subscribe foo + set reply [$rd1 read] + assert {$reply eq "subscribe foo 1"} + + set omem 0 + set start_time 0 + set time_elapsed 0 + set last_under_limit_time [clock milliseconds] + while 1 { + r publish foo [string repeat "x" 1000] + set clients [split [r client list] "\r\n"] + set c [lsearch -inline $clients *name=test_client*] + if {$start_time != 0} { + set time_elapsed [expr {[clock milliseconds]-$start_time}] + # Make sure test isn't taking too long + assert {$time_elapsed <= [expr $soft_limit_time+3000]} + } + if {$wait_for_timeout && $c == ""} { + # Make sure we're disconnected when we reach the soft limit + assert {$omem >= 100000 && $time_elapsed >= $soft_limit_time} + break + } else { + assert {[regexp {omem=([0-9]+)} $c - omem]} + } + if {$omem > 100000} { + if {$start_time == 0} {set start_time $last_under_limit_time} + if {!$wait_for_timeout && $time_elapsed >= [expr $soft_limit_time-1000]} break + # Slow down loop when omem has reached the limit. + after 10 + } else { + # if the OS socket buffers swallowed what we previously filled, reset the start timer. + set start_time 0 + set last_under_limit_time [clock milliseconds] + } } - r publish foo [string repeat "x" 1000] - set clients [split [r client list] "\r\n"] - set c [split [lindex $clients 1] " "] - if {![regexp {omem=([0-9]+)} $c - omem]} break - if {$omem > 100000} { - if {$start_time == 0} {set start_time [clock seconds]} - set time_elapsed [expr {[clock seconds]-$start_time}] - if {$time_elapsed >= 10} break + + if {!$wait_for_timeout} { + # After we completely stopped the traffic, wait for soft limit to time out + set timeout [expr {$soft_limit_time+1500 - ([clock milliseconds]-$start_time)}] + wait_for_condition [expr $timeout/10] 10 { + [lsearch [split [r client list] "\r\n"] *name=test_client*] == -1 + } else { + fail "Soft limit timed out but client still connected" + } } + + $rd1 close } - assert {$omem >= 100000 && $time_elapsed < 6} - $rd1 close } test {No response for single command if client output buffer hard limit is enforced} { From 435511f7c329ac1c9c69d7a724f80164b7478864 Mon Sep 17 00:00:00 2001 From: Wang Yuan Date: Thu, 6 May 2021 15:52:11 +0800 Subject: [PATCH 02/79] Fix wrong COW memory in log (#8917) Always 0 MB of memory used by copy-on-write, introduced in #8645. (cherry picked from commit 81e2d7272b784273099fecd85b15473277296771) --- src/childinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/childinfo.c b/src/childinfo.c index 4f0a42001..5e94a4f2c 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -93,7 +93,7 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, if (cow) { serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE, "%s: %zu MB of memory used by copy-on-write", - pname, data.cow / (1024 * 1024)); + pname, cow / (1024 * 1024)); } } From f8e272f765eedce83949fa1791f0873211544d85 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 6 May 2021 17:45:49 +0300 Subject: [PATCH 03/79] fix redis-benchmark to ignore unsupported configs (#8916) Redis Enterprise supports the CONFIG GET command, but it replies with am empty array since the save and appendonly configs are not supported. before this fix redis-benchmark would segfault for trying to access the error string on an array type reply. see #8869 (cherry picked from commit 4d1094e8be3150b92b3e96d3a743c66b1a95988a) --- src/redis-benchmark.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index fa024d44f..8d510d7da 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -368,9 +368,10 @@ fail: if (hostsocket == NULL) fprintf(stderr, "%s:%d\n", ip, port); else fprintf(stderr, "%s\n", hostsocket); int abort_test = 0; - if (!strncmp(reply->str,"NOAUTH",5) || - !strncmp(reply->str,"WRONGPASS",9) || - !strncmp(reply->str,"NOPERM",5)) + if (reply && reply->type == REDIS_REPLY_ERROR && + (!strncmp(reply->str,"NOAUTH",5) || + !strncmp(reply->str,"WRONGPASS",9) || + !strncmp(reply->str,"NOPERM",5))) abort_test = 1; freeReplyObject(reply); redisFree(c); From eca5e75b2a8bf7e4319f214ddc0e3b5e9f4522b8 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Mon, 10 May 2021 17:08:43 +0300 Subject: [PATCH 04/79] Fix crash unlinking a stream with groups rax and no groups (#8932) When estimating the effort for unlink, we try to compute the effort of the first group and extrapolate. If there's a groups rax that's empty, there'a an assertion. reproduce: xadd s * a b xgroup create s bla $ xgroup destroy s bla unlink s (cherry picked from commit 97108845e2ae7661e5091c817cb03459ec81ea8c) --- src/lazyfree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lazyfree.c b/src/lazyfree.c index a2cf2c3ed..180a4349b 100644 --- a/src/lazyfree.c +++ b/src/lazyfree.c @@ -109,7 +109,7 @@ size_t lazyfreeGetFreeEffort(robj *key, robj *obj) { /* Every consumer group is an allocation and so are the entries in its * PEL. We use size of the first group's PEL as an estimate for all * others. */ - if (s->cgroups) { + if (s->cgroups && raxSize(s->cgroups)) { raxIterator ri; streamCG *cg; raxStart(&ri,s->cgroups); From 2adf90348f164b5143fbded6bc8fb6753d714609 Mon Sep 17 00:00:00 2001 From: patpatbear Date: Sat, 15 May 2021 08:43:09 -0400 Subject: [PATCH 05/79] sinterstore: add missing keyspace del event when any source set not exists. (#8949) this patch fixes sinterstore by add missing keyspace del event when any source set not exists. Co-authored-by: srzhao (cherry picked from commit 46d9f31e94355ec15b95418377677bcf75839bc9) --- src/t_set.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/t_set.c b/src/t_set.c index b655b716d..bf250baa2 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -869,6 +869,7 @@ void sinterGenericCommand(client *c, robj **setkeys, if (dstkey) { if (dbDelete(c->db,dstkey)) { signalModifiedKey(c,c->db,dstkey); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id); server.dirty++; } addReply(c,shared.czero); From f8938d868b0d354eb7ad34baba86c60ca0524873 Mon Sep 17 00:00:00 2001 From: Madelyn Olson <34459052+madolson@users.noreply.github.com> Date: Wed, 19 May 2021 08:23:54 -0700 Subject: [PATCH 06/79] Hide migrate command from slowlog if they include auth (#8859) Redact commands that include sensitive data from slowlog and monitor (cherry picked from commit a59e75a475782d86d7ce2b5b9c6f5bb4a5ef0bd6) --- src/acl.c | 13 ++++++---- src/cluster.c | 3 +++ src/config.c | 2 +- src/multi.c | 14 ++--------- src/networking.c | 15 ++++++++---- src/replication.c | 1 + src/scripting.c | 6 +++++ src/server.c | 42 ++++++++++++++++---------------- src/server.h | 5 ++-- tests/unit/introspection.tcl | 46 ++++++++++++++++++++++++++++++++++++ tests/unit/other.tcl | 3 --- tests/unit/slowlog.tcl | 25 ++++++++++++++++---- 12 files changed, 122 insertions(+), 53 deletions(-) diff --git a/src/acl.c b/src/acl.c index 6a2ade646..86f73fe7e 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1892,10 +1892,6 @@ void addACLLogEntry(client *c, int reason, int argpos, sds username) { void aclCommand(client *c) { char *sub = c->argv[1]->ptr; if (!strcasecmp(sub,"setuser") && c->argc >= 3) { - /* Consider information about passwords or permissions - * to be sensitive, which will be the arguments for this - * subcommand. */ - preventCommandLogging(c); sds username = c->argv[2]->ptr; /* Check username validity. */ if (ACLStringHasSpaces(username,sdslen(username))) { @@ -1912,6 +1908,12 @@ void aclCommand(client *c) { user *u = ACLGetUserByName(username,sdslen(username)); if (u) ACLCopyUser(tempu, u); + /* Initially redact all of the arguments to not leak any information + * about the user. */ + for (int j = 2; j < c->argc; j++) { + redactClientCommandArgument(c, j); + } + for (int j = 3; j < c->argc; j++) { if (ACLSetUser(tempu,c->argv[j]->ptr,sdslen(c->argv[j]->ptr)) != C_OK) { const char *errmsg = ACLSetUserStringError(); @@ -2245,6 +2247,8 @@ void authCommand(client *c) { addReplyErrorObject(c,shared.syntaxerr); return; } + /* Always redact the second argument */ + redactClientCommandArgument(c, 1); /* Handle the two different forms here. The form with two arguments * will just use "default" as username. */ @@ -2264,6 +2268,7 @@ void authCommand(client *c) { } else { username = c->argv[1]; password = c->argv[2]; + redactClientCommandArgument(c, 2); } if (ACLAuthenticateUser(c,username,password) == C_OK) { diff --git a/src/cluster.c b/src/cluster.c index ba21024be..28650b327 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5361,13 +5361,16 @@ void migrateCommand(client *c) { } j++; password = c->argv[j]->ptr; + redactClientCommandArgument(c,j); } else if (!strcasecmp(c->argv[j]->ptr,"auth2")) { if (moreargs < 2) { addReplyErrorObject(c,shared.syntaxerr); return; } username = c->argv[++j]->ptr; + redactClientCommandArgument(c,j); password = c->argv[++j]->ptr; + redactClientCommandArgument(c,j); } else if (!strcasecmp(c->argv[j]->ptr,"keys")) { if (sdslen(c->argv[3]->ptr) != 0) { addReplyError(c, diff --git a/src/config.c b/src/config.c index 9861c5f52..e0eec2f67 100644 --- a/src/config.c +++ b/src/config.c @@ -726,7 +726,7 @@ void configSetCommand(client *c) { (config->alias && !strcasecmp(c->argv[2]->ptr,config->alias)))) { if (config->flags & SENSITIVE_CONFIG) { - preventCommandLogging(c); + redactClientCommandArgument(c,3); } if (!config->interface.set(config->data,o->ptr,1,&errstr)) { goto badfmt; diff --git a/src/multi.c b/src/multi.c index 902c919c7..5fb37098a 100644 --- a/src/multi.c +++ b/src/multi.c @@ -153,8 +153,7 @@ void execCommandAbort(client *c, sds error) { /* Send EXEC to clients waiting data from MONITOR. We did send a MULTI * already, and didn't send any of the queued commands, now we'll just send * EXEC so it is clear that the transaction is over. */ - if (listLength(server.monitors) && !server.loading) - replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); } void execCommand(client *c) { @@ -179,7 +178,7 @@ void execCommand(client *c) { addReply(c, c->flags & CLIENT_DIRTY_EXEC ? shared.execaborterr : shared.nullarray[c->resp]); discardTransaction(c); - goto handle_monitor; + return; } uint64_t old_flags = c->flags; @@ -266,15 +265,6 @@ void execCommand(client *c) { } server.in_exec = 0; - -handle_monitor: - /* Send EXEC to clients waiting data from MONITOR. We do it here - * since the natural order of commands execution is actually: - * MUTLI, EXEC, ... commands inside transaction ... - * Instead EXEC is flagged as CMD_SKIP_MONITOR in the command - * table, and we do it here with correct ordering. */ - if (listLength(server.monitors) && !server.loading) - replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); } /* ===================== WATCH (CAS alike for MULTI/EXEC) =================== diff --git a/src/networking.c b/src/networking.c index 14e94b84c..9de105982 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1663,9 +1663,6 @@ void resetClient(client *c) { c->flags |= CLIENT_REPLY_SKIP; c->flags &= ~CLIENT_REPLY_SKIP_NEXT; } - - /* Always clear the prevent logging field. */ - c->flags &= ~CLIENT_PREVENT_LOGGING; } /* This function is used when we want to re-enter the event loop but there @@ -2967,7 +2964,8 @@ void helloCommand(client *c) { int moreargs = (c->argc-1) - j; const char *opt = c->argv[j]->ptr; if (!strcasecmp(opt,"AUTH") && moreargs >= 2) { - preventCommandLogging(c); + redactClientCommandArgument(c, j+1); + redactClientCommandArgument(c, j+2); if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) { addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled."); return; @@ -3054,6 +3052,15 @@ static void retainOriginalCommandVector(client *c) { } } +/* Redact a given argument to prevent it from being shown + * in the slowlog. This information is stored in the + * original_argv array. */ +void redactClientCommandArgument(client *c, int argc) { + retainOriginalCommandVector(c); + decrRefCount(c->argv[argc]); + c->original_argv[argc] = shared.redacted; +} + /* Rewrite the command vector of the client. All the new objects ref count * is incremented. The old command vector is freed, and the old objects * ref count is decremented. */ diff --git a/src/replication.c b/src/replication.c index 8177eb073..932ce1dee 100644 --- a/src/replication.c +++ b/src/replication.c @@ -377,6 +377,7 @@ void replicationFeedSlavesFromMasterStream(list *slaves, char *buf, size_t bufle } void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, int argc) { + if (!(listLength(server.monitors) && !server.loading)) return; listNode *ln; listIter li; int j; diff --git a/src/scripting.c b/src/scripting.c index dbbd50eaf..b0602f7df 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1690,6 +1690,9 @@ void evalGenericCommand(client *c, int evalsha) { } void evalCommand(client *c) { + /* Explicitly feed monitor here so that lua commands appear after their + * script command. */ + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); if (!(c->flags & CLIENT_LUA_DEBUG)) evalGenericCommand(c,0); else @@ -1697,6 +1700,9 @@ void evalCommand(client *c) { } void evalShaCommand(client *c) { + /* Explicitly feed monitor here so that lua commands appear after their + * script command. */ + replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); if (sdslen(c->argv[1]->ptr) != 40) { /* We know that a match is not possible if the provided SHA is * not the right length. So we return an error ASAP, this way diff --git a/src/server.c b/src/server.c index 9541aada7..516bcb114 100644 --- a/src/server.c +++ b/src/server.c @@ -706,7 +706,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"auth",authCommand,-2, - "no-auth no-script ok-loading ok-stale fast no-monitor no-slowlog @connection", + "no-auth no-script ok-loading ok-stale fast @connection", 0,NULL,0,0,0,0,0,0}, /* We don't allow PING during loading since in Redis PING is used as @@ -749,7 +749,7 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"exec",execCommand,1, - "no-script no-monitor no-slowlog ok-loading ok-stale @transaction", + "no-script no-slowlog ok-loading ok-stale @transaction", 0,NULL,0,0,0,0,0,0}, {"discard",discardCommand,1, @@ -901,17 +901,21 @@ struct redisCommand redisCommandTable[] = { 0,NULL,0,0,0,0,0,0}, {"hello",helloCommand,-1, - "no-auth no-script fast no-monitor ok-loading ok-stale @connection", + "no-auth no-script fast ok-loading ok-stale @connection", 0,NULL,0,0,0,0,0,0}, /* EVAL can modify the dataset, however it is not flagged as a write - * command since we do the check while running commands from Lua. */ + * command since we do the check while running commands from Lua. + * + * EVAL and EVALSHA also feed monitors before the commands are executed, + * as opposed to after. + */ {"eval",evalCommand,-3, - "no-script may-replicate @scripting", + "no-script no-monitor may-replicate @scripting", 0,evalGetKeys,0,0,0,0,0,0}, {"evalsha",evalShaCommand,-3, - "no-script may-replicate @scripting", + "no-script no-monitor may-replicate @scripting", 0,evalGetKeys,0,0,0,0,0,0}, {"slowlog",slowlogCommand,-2, @@ -2604,6 +2608,7 @@ void createSharedObjects(void) { shared.getack = createStringObject("GETACK",6); shared.special_asterick = createStringObject("*",1); shared.special_equals = createStringObject("=",1); + shared.redacted = makeObjectShared(createStringObject("(redacted)",10)); for (j = 0; j < OBJ_SHARED_INTEGERS; j++) { shared.integers[j] = @@ -3625,12 +3630,6 @@ void preventCommandPropagation(client *c) { c->flags |= CLIENT_PREVENT_PROP; } -/* Avoid logging any information about this client's arguments - * since they contain sensitive information. */ -void preventCommandLogging(client *c) { - c->flags |= CLIENT_PREVENT_LOGGING; -} - /* AOF specific version of preventCommandPropagation(). */ void preventCommandAOF(client *c) { c->flags |= CLIENT_PREVENT_AOF_PROP; @@ -3644,7 +3643,7 @@ void preventCommandReplication(client *c) { /* Log the last command a client executed into the slowlog. */ void slowlogPushCurrentCommand(client *c, struct redisCommand *cmd, ustime_t duration) { /* Some commands may contain sensitive data that should not be available in the slowlog. */ - if ((c->flags & CLIENT_PREVENT_LOGGING) || (cmd->flags & CMD_SKIP_SLOWLOG)) + if (cmd->flags & CMD_SKIP_SLOWLOG) return; /* If command argument vector was rewritten, use the original @@ -3700,15 +3699,6 @@ void call(client *c, int flags) { server.fixed_time_expire++; - /* Send the command to clients in MONITOR mode if applicable. - * Administrative commands are considered too dangerous to be shown. */ - if (listLength(server.monitors) && - !server.loading && - !(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN))) - { - replicationFeedMonitors(c,server.monitors,c->db->id,c->argv,c->argc); - } - /* Initialization: clear the flags that must be set by the command on * demand, and initialize the array for additional commands propagation. */ c->flags &= ~(CLIENT_FORCE_AOF|CLIENT_FORCE_REPL|CLIENT_PREVENT_PROP); @@ -3774,6 +3764,14 @@ void call(client *c, int flags) { if ((flags & CMD_CALL_SLOWLOG) && !(c->flags & CLIENT_BLOCKED)) slowlogPushCurrentCommand(c, real_cmd, duration); + /* Send the command to clients in MONITOR mode if applicable. + * Administrative commands are considered too dangerous to be shown. */ + if (!(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN))) { + robj **argv = c->original_argv ? c->original_argv : c->argv; + int argc = c->original_argv ? c->original_argc : c->argc; + replicationFeedMonitors(c,server.monitors,c->db->id,argv,argc); + } + /* Clear the original argv. * If the client is blocked we will handle slowlog when it is unblocked. */ if (!(c->flags & CLIENT_BLOCKED)) diff --git a/src/server.h b/src/server.h index ee87ecacf..86c926805 100644 --- a/src/server.h +++ b/src/server.h @@ -279,7 +279,6 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; and AOF client */ #define CLIENT_REPL_RDBONLY (1ULL<<42) /* This client is a replica that only wants RDB without replication buffer. */ -#define CLIENT_PREVENT_LOGGING (1ULL<<43) /* Prevent logging of command to slowlog */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ @@ -986,7 +985,7 @@ struct sharedObjectsStruct { *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, *time, *pxat, *px, *retrycount, *force, *justid, *lastid, *ping, *setid, *keepttl, *load, *createconsumer, - *getack, *special_asterick, *special_equals, *default_username, + *getack, *special_asterick, *special_equals, *default_username, *redacted, *select[PROTO_SHARED_SELECT_CMDS], *integers[OBJ_SHARED_INTEGERS], *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*\r\n" */ @@ -1865,6 +1864,7 @@ sds getAllClientsInfoString(int type); void rewriteClientCommandVector(client *c, int argc, ...); void rewriteClientCommandArgument(client *c, int i, robj *newval); void replaceClientCommandVector(client *c, int argc, robj **argv); +void redactClientCommandArgument(client *c, int argc); unsigned long getClientOutputBufferMemoryUsage(client *c); int freeClientsInAsyncFreeQueue(void); int closeClientOnOutputBufferLimitReached(client *c, int async); @@ -2213,7 +2213,6 @@ void redisOpArrayInit(redisOpArray *oa); void redisOpArrayFree(redisOpArray *oa); void forceCommandPropagation(client *c, int flags); void preventCommandPropagation(client *c); -void preventCommandLogging(client *c); void preventCommandAOF(client *c); void preventCommandReplication(client *c); void slowlogPushCurrentCommand(client *c, struct redisCommand *cmd, ustime_t duration); diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 7ce89aa01..0c78edf20 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -31,6 +31,52 @@ start_server {tags {"introspection"}} { assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] } + test {MONITOR supports redacting command arguments} { + set rd [redis_deferring_client] + $rd monitor + $rd read ; # Discard the OK + + r migrate [srv 0 host] [srv 0 port] key 9 5000 + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH user + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH2 user password + catch {r auth not-real} _ + catch {r auth not-real not-a-password} _ + catch {r hello 2 AUTH not-real not-a-password} _ + + assert_match {*"key"*"9"*"5000"*} [$rd read] + assert_match {*"key"*"9"*"5000"*"(redacted)"*} [$rd read] + assert_match {*"key"*"9"*"5000"*"(redacted)"*"(redacted)"*} [$rd read] + assert_match {*"auth"*"(redacted)"*} [$rd read] + assert_match {*"auth"*"(redacted)"*"(redacted)"*} [$rd read] + assert_match {*"hello"*"2"*"AUTH"*"(redacted)"*"(redacted)"*} [$rd read] + $rd close + } + + test {MONITOR correctly handles multi-exec cases} { + set rd [redis_deferring_client] + $rd monitor + $rd read ; # Discard the OK + + # Make sure multi-exec statements are ordered + # correctly + r multi + r set foo bar + r exec + assert_match {*"multi"*} [$rd read] + assert_match {*"set"*"foo"*"bar"*} [$rd read] + assert_match {*"exec"*} [$rd read] + + # Make sure we close multi statements on errors + r multi + catch {r syntax error} _ + catch {r exec} _ + + assert_match {*"multi"*} [$rd read] + assert_match {*"exec"*} [$rd read] + + $rd close + } + test {CLIENT GETNAME should return NIL if name is not assigned} { r client getname } {} diff --git a/tests/unit/other.tcl b/tests/unit/other.tcl index a6b0d0132..293ee7e95 100644 --- a/tests/unit/other.tcl +++ b/tests/unit/other.tcl @@ -266,9 +266,6 @@ start_server {overrides {save ""} tags {"other"}} { assert_equal [$rd read] "OK" $rd reset - - # skip reset ouptut - $rd read assert_equal [$rd read] "RESET" assert_no_match {*flags=O*} [r client list] diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl index eb9dfc65d..9f6e248e9 100644 --- a/tests/unit/slowlog.tcl +++ b/tests/unit/slowlog.tcl @@ -45,18 +45,35 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { r config set slowlog-log-slower-than 0 r slowlog reset r config set masterauth "" - r acl setuser slowlog-test-user + r acl setuser slowlog-test-user +get +set r config set slowlog-log-slower-than 0 r config set slowlog-log-slower-than 10000 set slowlog_resp [r slowlog get] # Make sure normal configs work, but the two sensitive - # commands are omitted - assert_equal 2 [llength $slowlog_resp] - assert_equal {slowlog reset} [lindex [lindex [r slowlog get] 1] 3] + # commands are omitted or redacted + assert_equal 4 [llength $slowlog_resp] + assert_equal {slowlog reset} [lindex [lindex [r slowlog get] 3] 3] + assert_equal {config set masterauth (redacted)} [lindex [lindex [r slowlog get] 2] 3] + assert_equal {acl setuser (redacted) (redacted) (redacted)} [lindex [lindex [r slowlog get] 1] 3] assert_equal {config set slowlog-log-slower-than 0} [lindex [lindex [r slowlog get] 0] 3] } + test {SLOWLOG - Some commands can redact sensitive fields} { + r config set slowlog-log-slower-than 0 + r slowlog reset + r migrate [srv 0 host] [srv 0 port] key 9 5000 + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH user + r migrate [srv 0 host] [srv 0 port] key 9 5000 AUTH2 user password + + r config set slowlog-log-slower-than 10000 + # Make sure all 3 commands were logged, but the sensitive fields are omitted + assert_equal 4 [llength [r slowlog get]] + assert_match {* key 9 5000} [lindex [lindex [r slowlog get] 2] 3] + assert_match {* key 9 5000 AUTH (redacted)} [lindex [lindex [r slowlog get] 1] 3] + assert_match {* key 9 5000 AUTH2 (redacted) (redacted)} [lindex [lindex [r slowlog get] 0] 3] + } + test {SLOWLOG - Rewritten commands are logged as their original command} { r config set slowlog-log-slower-than 0 From efb7a5c630f12222a2071ec995c4dee82d9f9ebb Mon Sep 17 00:00:00 2001 From: Wen Hui Date: Sun, 23 May 2021 07:31:01 -0400 Subject: [PATCH 07/79] [SENTINEL] reset sentinel-user/pass to NULL when user config with empty string (#8958) (cherry picked from commit ae6f58690b91010d003cdf5552d74b8e5b428d53) --- src/sentinel.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 2d81d98ac..7b387ef83 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -3175,11 +3175,13 @@ void sentinelConfigSetCommand(client *c) { sentinel.announce_port = numval; } else if (!strcasecmp(o->ptr, "sentinel-user")) { sdsfree(sentinel.sentinel_auth_user); - sentinel.sentinel_auth_user = sdsnew(val->ptr); + sentinel.sentinel_auth_user = sdslen(val->ptr) == 0 ? + sdsdup(val->ptr) : NULL; drop_conns = 1; } else if (!strcasecmp(o->ptr, "sentinel-pass")) { sdsfree(sentinel.sentinel_auth_pass); - sentinel.sentinel_auth_pass = sdsnew(val->ptr); + sentinel.sentinel_auth_pass = sdslen(val->ptr) == 0 ? + sdsdup(val->ptr) : NULL; drop_conns = 1; } else { addReplyErrorFormat(c, "Invalid argument '%s' to SENTINEL CONFIG SET", From eb6d4da231486d24229ae539f4222971d81e75eb Mon Sep 17 00:00:00 2001 From: Wen Hui Date: Mon, 24 May 2021 01:36:51 -0400 Subject: [PATCH 08/79] fix sentinel test failure (#8983) fix for recent breakage from #8958, did a mistake when updating the pr. (cherry picked from commit be6ce8a92a9acbecfaaa6c57a45037fc1018fefe) --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 7b387ef83..a9d123ebc 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -3176,12 +3176,12 @@ void sentinelConfigSetCommand(client *c) { } else if (!strcasecmp(o->ptr, "sentinel-user")) { sdsfree(sentinel.sentinel_auth_user); sentinel.sentinel_auth_user = sdslen(val->ptr) == 0 ? - sdsdup(val->ptr) : NULL; + NULL : sdsdup(val->ptr); drop_conns = 1; } else if (!strcasecmp(o->ptr, "sentinel-pass")) { sdsfree(sentinel.sentinel_auth_pass); sentinel.sentinel_auth_pass = sdslen(val->ptr) == 0 ? - sdsdup(val->ptr) : NULL; + NULL : sdsdup(val->ptr); drop_conns = 1; } else { addReplyErrorFormat(c, "Invalid argument '%s' to SENTINEL CONFIG SET", From bd2785d18bb36d829f9f6e28b982b6512bbba51c Mon Sep 17 00:00:00 2001 From: YaacovHazan <31382944+YaacovHazan@users.noreply.github.com> Date: Wed, 26 May 2021 14:51:53 +0300 Subject: [PATCH 09/79] unregister AE_READABLE from the read pipe in backgroundSaveDoneHandlerSocket (#8991) In diskless replication, we create a read pipe for the RDB, between the child and the parent. When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). Otherwise, next time we will use the same fd, the registration will be fail (panic), because we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl (cherry picked from commit 501d7755831527b4237f9ed6050ec84203934e4d) --- src/rdb.c | 1 + tests/integration/replication.tcl | 39 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/rdb.c b/src/rdb.c index 6f2f5165e..3c5397d5f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2691,6 +2691,7 @@ static void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) { } if (server.rdb_child_exit_pipe!=-1) close(server.rdb_child_exit_pipe); + aeDeleteFileEvent(server.el, server.rdb_pipe_read, AE_READABLE); close(server.rdb_pipe_read); server.rdb_child_exit_pipe = -1; server.rdb_pipe_read = -1; diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 1a089ef4b..fcef2e9e5 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -774,6 +774,45 @@ test "diskless replication child being killed is collected" { } } +test "diskless replication read pipe cleanup" { + # In diskless replication, we create a read pipe for the RDB, between the child and the parent. + # When we close this pipe (fd), the read handler also needs to be removed from the event loop (if it still registered). + # Otherwise, next time we will use the same fd, the registration will be fail (panic), because + # we will use EPOLL_CTL_MOD (the fd still register in the event loop), on fd that already removed from epoll_ctl + start_server {tags {"repl"}} { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set master_pid [srv 0 pid] + $master config set repl-diskless-sync yes + $master config set repl-diskless-sync-delay 0 + + # put enough data in the db, and slowdown the save, to keep the parent busy at the read process + $master config set rdb-key-save-delay 100000 + $master debug populate 20000 test 10000 + $master config set rdbcompression no + start_server {} { + set replica [srv 0 client] + set loglines [count_log_lines 0] + $replica config set repl-diskless-load swapdb + $replica replicaof $master_host $master_port + + # wait for the replicas to start reading the rdb + wait_for_log_messages 0 {"*Loading DB in memory*"} $loglines 800 10 + + set loglines [count_log_lines 0] + # send FLUSHALL so the RDB child will be killed + $master flushall + + # wait for another RDB child process to be started + wait_for_log_messages -1 {"*Background RDB transfer started by pid*"} $loglines 800 10 + + # make sure master is alive + $master ping + } + } +} + test {replicaof right after disconnection} { # this is a rare race condition that was reproduced sporadically by the psync2 unit. # see details in #7205 From e55cc0e9f4c2e95ba9fb24debd9295c6889c2b5a Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 1 Jun 2021 09:12:45 +0300 Subject: [PATCH 10/79] Fix integer overflow in STRALGO LCS (CVE-2021-32625) (#9011) An integer overflow bug in Redis version 6.0 or newer can be exploited using the STRALGO LCS command to corrupt the heap and potentially result with remote code execution. This is a result of an incomplete fix by CVE-2021-29477. (cherry picked from commit 1ddecf1958924b178b76a31d989ef1e05af81964) --- src/t_string.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/t_string.c b/src/t_string.c index 490d5983a..587d3aeb8 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -797,6 +797,12 @@ void stralgoLCS(client *c) { goto cleanup; } + /* Detect string truncation or later overflows. */ + if (sdslen(a) >= UINT32_MAX-1 || sdslen(b) >= UINT32_MAX-1) { + addReplyError(c, "String too long for LCS"); + goto cleanup; + } + /* Compute the LCS using the vanilla dynamic programming technique of * building a table of LCS(x,y) substrings. */ uint32_t alen = sdslen(a); @@ -805,9 +811,19 @@ void stralgoLCS(client *c) { /* Setup an uint32_t array to store at LCS[i,j] the length of the * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so * we index it as LCS[j+(blen+1)*j] */ - uint32_t *lcs = zmalloc((size_t)(alen+1)*(blen+1)*sizeof(uint32_t)); #define LCS(A,B) lcs[(B)+((A)*(blen+1))] + /* Try to allocate the LCS table, and abort on overflow or insufficient memory. */ + unsigned long long lcssize = (unsigned long long)(alen+1)*(blen+1); /* Can't overflow due to the size limits above. */ + unsigned long long lcsalloc = lcssize * sizeof(uint32_t); + uint32_t *lcs = NULL; + if (lcsalloc < SIZE_MAX && lcsalloc / lcssize == sizeof(uint32_t)) + lcs = ztrymalloc(lcsalloc); + if (!lcs) { + addReplyError(c, "Insufficient memory"); + goto cleanup; + } + /* Start building the LCS table. */ for (uint32_t i = 0; i <= alen; i++) { for (uint32_t j = 0; j <= blen; j++) { From 8728f68f6648cc416d8882424470a6063cb32c74 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 1 Jun 2021 09:11:53 +0300 Subject: [PATCH 11/79] Redis 6.2.4 --- 00-RELEASENOTES | 25 +++++++++++++++++++++++++ src/version.h | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 4f6cb9978..9411714e4 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,31 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.2.4 Released Tue July 1 12:00:00 IST 2021 +================================================================================ + +Upgrade urgency: SECURITY, Contains fixes to security issues that affect +authenticated client connections. MODERATE otherwise. + +Fix integer overflow in STRALGO LCS (CVE-2021-32625) +An integer overflow bug in Redis version 6.0 or newer can be exploited using the +STRALGO LCS command to corrupt the heap and potentially result with remote code +execution. This is a result of an incomplete fix by CVE-2021-29477. + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix crash after a diskless replication fork child is terminated (#8991) +* Fix redis-benchmark crash on unsupported configs (#8916) + +Other bug fixes: +* Fix crash in UNLINK on a stream key with deleted consumer groups (#8932) +* SINTERSTORE: Add missing keyspace del event when none of the sources exist (#8949) +* Sentinel: Fix CONFIG SET of empty string sentinel-user/sentinel-pass configs (#8958) +* Enforce client output buffer soft limit when no traffic (#8833) + +Improvements: +* Hide AUTH passwords in MIGRATE command from slowlog (#8859) + ================================================================================ Redis 6.2.3 Released Mon May 3 19:00:00 IST 2021 ================================================================================ diff --git a/src/version.h b/src/version.h index b87f2b9c3..c355ecfed 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.2.3" -#define REDIS_VERSION_NUM 0x00060203 +#define REDIS_VERSION "6.2.4" +#define REDIS_VERSION_NUM 0x00060204 From 78d65970129228069cc4dc662eb7c2a731958157 Mon Sep 17 00:00:00 2001 From: M Sazzadul Hoque <7600764+sazzad16@users.noreply.github.com> Date: Tue, 1 Jun 2021 22:03:06 +0600 Subject: [PATCH 12/79] Fix 6.2.4 release month to June (#9027) --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 9411714e4..3fd620cf0 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -12,7 +12,7 @@ SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- ================================================================================ -Redis 6.2.4 Released Tue July 1 12:00:00 IST 2021 +Redis 6.2.4 Released Tue June 1 12:00:00 IST 2021 ================================================================================ Upgrade urgency: SECURITY, Contains fixes to security issues that affect From 9562821f12fc0211837c213218f3c79219623af8 Mon Sep 17 00:00:00 2001 From: Maxim Galushka Date: Mon, 21 Jun 2021 09:01:31 +0100 Subject: [PATCH 13/79] redis-cli: support for REDIS_REPLY_SET in CSV and RAW output. (#7338) Fixes #6792. Added support of REDIS_REPLY_SET in raw and csv output of `./redis-cli` Test: run commands to test: ./redis-cli -3 --csv COMMAND ./redis-cli -3 --raw COMMAND Now they are returning resuts, were failing with: "Unknown reply type: 10" before the change. (cherry picked from commit 96bb078577ce2b0d093c873faae5d3ecca26a1de) --- src/redis-cli.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 81be58b17..8e049f186 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1129,6 +1129,7 @@ static sds cliFormatReplyRaw(redisReply *r) { case REDIS_REPLY_DOUBLE: out = sdscatprintf(out,"%s",r->str); break; + case REDIS_REPLY_SET: case REDIS_REPLY_ARRAY: case REDIS_REPLY_PUSH: for (i = 0; i < r->elements; i++) { @@ -1187,6 +1188,7 @@ static sds cliFormatReplyCSV(redisReply *r) { out = sdscat(out,r->integer ? "true" : "false"); break; case REDIS_REPLY_ARRAY: + case REDIS_REPLY_SET: case REDIS_REPLY_PUSH: case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */ for (i = 0; i < r->elements; i++) { From f1ff72c7a3f6aa8255c85eecdea9097618d0dfde Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 22 Jun 2021 07:35:59 +0300 Subject: [PATCH 14/79] Fix race in client side tracking (#9116) The `Tracking gets notification of expired keys` test in tracking.tcl used to hung in valgrind CI quite a lot. It turns out the reason is that with valgrind and a busy machine, the server cron active expire cycle could easily run in the same event loop as the command that created `mykey`, so that when they key got expired, there were two change events to broadcast, one that set the key and one that expired it, but since we used raxTryInsert, the client that was associated with the "last" change was the one that created the key, so the NOLOOP filtered that event. This commit adds a test that reproduces the problem by using lazy expire in a multi-exec which makes sure the key expires in the same event loop as the one that added it. (cherry picked from commit 9b564b525d8ce88295ec14ffdc3bede7e5f5c33e) --- src/tracking.c | 2 +- tests/unit/tracking.tcl | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/tracking.c b/src/tracking.c index a11e4b7d7..a57f062c6 100644 --- a/src/tracking.c +++ b/src/tracking.c @@ -326,7 +326,7 @@ void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) { * tree. This way we know who was the client that did the last * change to the key, and can avoid sending the notification in the * case the client is in NOLOOP mode. */ - raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL); + raxInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL); } raxStop(&ri); } diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index 4c75b6f48..217a057dd 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -132,6 +132,22 @@ start_server {tags {"tracking network"}} { assert {$keys eq {mykey}} } + test {Tracking gets notification of lazy expired keys} { + r CLIENT TRACKING off + r CLIENT TRACKING on BCAST REDIRECT $redir_id NOLOOP + # Use multi-exec to expose a race where the key gets an two invalidations + # in the same event loop, once by the client so filtered by NOLOOP, and + # the second one by the lazy expire + r MULTI + r SET mykey{t} myval px 1 + r SET mykeyotherkey{t} myval ; # We should not get it + r DEBUG SLEEP 0.1 + r GET mykey{t} + r EXEC + set keys [lsort [lindex [$rd_redirection read] 2]] + assert {$keys eq {mykey{t}}} + } {} {needs:debug} + test {HELLO 3 reply is correct} { set reply [r HELLO 3] assert_equal [dict get $reply proto] 3 From 3687f210a13d3c90e6b684c21aa8a8de630a4256 Mon Sep 17 00:00:00 2001 From: Omer Shadmi <76992134+oshadmi@users.noreply.github.com> Date: Tue, 6 Jul 2021 08:21:17 +0300 Subject: [PATCH 15/79] Avoid exiting to allow diskless loading to recover from RDB short read on module AUX data (#9199) Currently a replica is able to recover from a short read (when diskless loading is enabled) and avoid crashing/exiting, replying to the master and then the rdb could be sent again by the master for another load attempt by the replica. There were a few scenarios that were not behaving similarly, such as when there is no end-of-file marker, or when module aux data failed to load, which should be allowed to occur due to a short read. (cherry picked from commit f06d782f5abcb30efb0117841232828ed3e129bf) --- src/rdb.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 3c5397d5f..e902388e6 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2484,8 +2484,10 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { int when_opcode = rdbLoadLen(rdb,NULL); int when = rdbLoadLen(rdb,NULL); if (rioGetReadError(rdb)) goto eoferr; - if (when_opcode != RDB_MODULE_OPCODE_UINT) + if (when_opcode != RDB_MODULE_OPCODE_UINT) { rdbReportReadError("bad when_opcode"); + goto eoferr; + } moduleType *mt = moduleTypeLookupModuleByID(moduleid); char name[10]; moduleTypeNameByID(name,moduleid); @@ -2509,7 +2511,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if (mt->aux_load(&io,moduleid&1023, when) != REDISMODULE_OK || io.error) { moduleTypeNameByID(name,moduleid); serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name); - exit(1); + goto eoferr; } if (io.ctx) { moduleFreeContext(io.ctx); @@ -2518,7 +2520,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { uint64_t eof = rdbLoadLen(rdb,NULL); if (eof != RDB_MODULE_OPCODE_EOF) { serverLog(LL_WARNING,"The RDB file contains module AUX data for the module '%s' that is not terminated by the proper module value EOF marker", name); - exit(1); + goto eoferr; } continue; } else { From 970302c00cec771ece4c5b8c41a4146b17b153d7 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Sun, 11 Jul 2021 18:00:17 +0800 Subject: [PATCH 16/79] Do not install a file event to send data to rewrite child when parent stop sending diff to child in aof rewrite. (#8767) In aof rewrite, when parent stop sending data to child, if there is new rewrite data, aofChildWriteDiffData write event will be installed. Then this event is issued and deletes the file event without do anyting. This will happen over and over again until aof rewrite finish. This bug used to waste a few system calls per excessive wake-up (epoll_ctl and epoll_wait) per cycle, each cycle triggered by receiving a write command from a client. (cherry picked from commit cb961d8c8e10ff3b619f1579a03336a15e9e6f45) --- src/aof.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/aof.c b/src/aof.c index 0d2971eac..566a98640 100644 --- a/src/aof.c +++ b/src/aof.c @@ -162,7 +162,9 @@ void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { /* Install a file event to send data to the rewrite child if there is * not one already. */ - if (aeGetFileEvents(server.el,server.aof_pipe_write_data_to_child) == 0) { + if (!server.aof_stop_sending_diff && + aeGetFileEvents(server.el,server.aof_pipe_write_data_to_child) == 0) + { aeCreateFileEvent(server.el, server.aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL); } From 6ae3673b5e4fd3d00cbc9d14e7c0953344b85f08 Mon Sep 17 00:00:00 2001 From: perryitay <85821686+perryitay@users.noreply.github.com> Date: Sun, 11 Jul 2021 13:17:23 +0300 Subject: [PATCH 17/79] Fail EXEC command in case a watched key is expired (#9194) There are two issues fixed in this commit: 1. we want to fail the EXEC command in case there is a watched key that's logically expired but not yet deleted by active expire or lazy expire. 2. we saw that currently cache time is update in every `call()` (including nested calls), this time is being also being use for the isKeyExpired comparison, we want to update the cache time only in the first call (execCommand) Co-authored-by: Oran Agra (cherry picked from commit ac8b1df8850cc80fbf9ce8c2fbde0c1d3a1b4e91) --- src/multi.c | 21 +++++++++++++++++++++ src/server.c | 10 +++++++--- src/server.h | 2 ++ tests/unit/multi.tcl | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/multi.c b/src/multi.c index 5fb37098a..3a157bee6 100644 --- a/src/multi.c +++ b/src/multi.c @@ -168,6 +168,11 @@ void execCommand(client *c) { return; } + /* EXEC with expired watched key is disallowed*/ + if (isWatchedKeyExpired(c)) { + c->flags |= (CLIENT_DIRTY_CAS); + } + /* Check if we need to abort the EXEC because: * 1) Some WATCHed key was touched. * 2) There was a previous error while queueing commands. @@ -342,6 +347,22 @@ void unwatchAllKeys(client *c) { } } +/* iterates over the watched_keys list and + * look for an expired key . */ +int isWatchedKeyExpired(client *c) { + listIter li; + listNode *ln; + watchedKey *wk; + if (listLength(c->watched_keys) == 0) return 0; + listRewind(c->watched_keys,&li); + while ((ln = listNext(&li))) { + wk = listNodeValue(ln); + if (keyIsExpired(wk->db, wk->key)) return 1; + } + + return 0; +} + /* "Touch" a key, so that if this key is being WATCHed by some client the * next EXEC will fail. */ void touchWatchedKey(redisDb *db, robj *key) { diff --git a/src/server.c b/src/server.c index 516bcb114..6c001b266 100644 --- a/src/server.c +++ b/src/server.c @@ -3697,8 +3697,6 @@ void call(client *c, int flags) { struct redisCommand *real_cmd = c->cmd; static long long prev_err_count; - server.fixed_time_expire++; - /* Initialization: clear the flags that must be set by the command on * demand, and initialize the array for additional commands propagation. */ c->flags &= ~(CLIENT_FORCE_AOF|CLIENT_FORCE_REPL|CLIENT_PREVENT_PROP); @@ -3708,7 +3706,13 @@ void call(client *c, int flags) { /* Call the command. */ dirty = server.dirty; prev_err_count = server.stat_total_error_replies; - updateCachedTime(0); + + /* Update cache time, in case we have nested calls we want to + * update only on the first call*/ + if (server.fixed_time_expire++ == 0) { + updateCachedTime(0); + } + elapsedStart(&call_timer); c->cmd->proc(c); const long duration = elapsedUs(call_timer); diff --git a/src/server.h b/src/server.h index 86c926805..64d08db60 100644 --- a/src/server.h +++ b/src/server.h @@ -1945,6 +1945,7 @@ void initClientMultiState(client *c); void freeClientMultiState(client *c); void queueMultiCommand(client *c); void touchWatchedKey(redisDb *db, robj *key); +int isWatchedKeyExpired(client *c); void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with); void discardTransaction(client *c); void flagTransaction(client *c); @@ -2318,6 +2319,7 @@ void initConfigValues(); /* db.c -- Keyspace access API */ int removeExpire(redisDb *db, robj *key); void propagateExpire(redisDb *db, robj *key, int lazy); +int keyIsExpired(redisDb *db, robj *key); int expireIfNeeded(redisDb *db, robj *key); long long getExpire(redisDb *db, robj *key); void setExpire(client *c, redisDb *db, robj *key, long long when); diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index e22b6d43d..d33f94515 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -121,6 +121,22 @@ start_server {tags {"multi"}} { r exec } {} + test {EXEC fail on lazy expired WATCHed key} { + r flushall + r debug set-active-expire 0 + + r del key + r set key 1 px 2 + r watch key + + after 100 + + r multi + r incr key + assert_equal [r exec] {} + r debug set-active-expire 1 + } {OK} {needs:debug} + test {After successful EXEC key is no longer watched} { r set x 30 r watch x From 4af7a28728643cc13727437676c029779af8ec76 Mon Sep 17 00:00:00 2001 From: YaacovHazan <31382944+YaacovHazan@users.noreply.github.com> Date: Thu, 20 May 2021 15:29:43 +0300 Subject: [PATCH 18/79] stabilize tests that involved with load handlers (#8967) When test stop 'load handler' by killing the process that generating the load, some commands that already in the input buffer, still might be processed by the server. This may cause some instability in tests, that count on that no more commands processed after we stop the `load handler' In this commit, new proc 'wait_load_handlers_disconnected' added, to verify that no more cammands from any 'load handler' prossesed, by checking that the clients who genreate the load is disconnceted. Also, replacing check of dbsize with wait_for_ofs_sync before comparing debug digest, as it would fail in case the last key the workload wrote was an overridden key (not a new one). Affected tests Race fix: - failover command to specific replica works - Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl - AOF rewrite during write load: RDB preamble=$rdbpre Cleanup and speedup: - Test replication with blocking lists and sorted sets operations - Test replication with parallel clients writing in different DBs - Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect (cherry picked from commit 32a2584e079a1b3c2d1e6649e38239381a73a459) --- tests/helpers/bg_block_op.tcl | 1 + tests/helpers/bg_complex_data.tcl | 1 + tests/helpers/gen_write_load.tcl | 1 + tests/integration/block-repl.tcl | 15 ++++----------- tests/integration/failover.tcl | 4 ++++ tests/integration/replication-4.tcl | 17 +++++------------ tests/integration/replication-psync.tcl | 17 +++++------------ tests/integration/replication.tcl | 15 ++++++--------- tests/support/util.tcl | 8 ++++++++ tests/unit/aofrw.tcl | 11 ++--------- 10 files changed, 37 insertions(+), 53 deletions(-) diff --git a/tests/helpers/bg_block_op.tcl b/tests/helpers/bg_block_op.tcl index c8b323308..dc4e1a999 100644 --- a/tests/helpers/bg_block_op.tcl +++ b/tests/helpers/bg_block_op.tcl @@ -12,6 +12,7 @@ set ::tlsdir "tests/tls" # blocking. proc bg_block_op {host port db ops tls} { set r [redis $host $port 0 $tls] + $r client setname LOAD_HANDLER $r select $db for {set j 0} {$j < $ops} {incr j} { diff --git a/tests/helpers/bg_complex_data.tcl b/tests/helpers/bg_complex_data.tcl index e888748a7..9c0044e7f 100644 --- a/tests/helpers/bg_complex_data.tcl +++ b/tests/helpers/bg_complex_data.tcl @@ -5,6 +5,7 @@ set ::tlsdir "tests/tls" proc bg_complex_data {host port db ops tls} { set r [redis $host $port 0 $tls] + $r client setname LOAD_HANDLER $r select $db createComplexDataset $r $ops } diff --git a/tests/helpers/gen_write_load.tcl b/tests/helpers/gen_write_load.tcl index cbf6651bd..568f5cde2 100644 --- a/tests/helpers/gen_write_load.tcl +++ b/tests/helpers/gen_write_load.tcl @@ -5,6 +5,7 @@ set ::tlsdir "tests/tls" proc gen_write_load {host port seconds tls} { set start_time [clock seconds] set r [redis $host $port 1 $tls] + $r client setname LOAD_HANDLER $r select 9 while 1 { $r set [expr rand()] [expr rand()] diff --git a/tests/integration/block-repl.tcl b/tests/integration/block-repl.tcl index 07eceb228..7c2ba840d 100644 --- a/tests/integration/block-repl.tcl +++ b/tests/integration/block-repl.tcl @@ -33,14 +33,9 @@ start_server {tags {"repl"}} { stop_bg_block_op $load_handle0 stop_bg_block_op $load_handle1 stop_bg_block_op $load_handle2 - set retry 10 - while {$retry && ([$master debug digest] ne [$slave debug digest])}\ - { - after 1000 - incr retry -1 - } - - if {[$master debug digest] ne [$slave debug digest]} { + wait_for_condition 100 100 { + [$master debug digest] == [$slave debug digest] + } else { set csv1 [csvdump r] set csv2 [csvdump {r -1}] set fd [open /tmp/repldump1.txt w] @@ -49,10 +44,8 @@ start_server {tags {"repl"}} { set fd [open /tmp/repldump2.txt w] puts -nonewline $fd $csv2 close $fd - puts "Master - Replica inconsistency" - puts "Run diff -u against /tmp/repldump*.txt for more info" + fail "Master - Replica inconsistency, Run diff -u against /tmp/repldump*.txt for more info" } - assert_equal [r debug digest] [r -1 debug digest] } } } diff --git a/tests/integration/failover.tcl b/tests/integration/failover.tcl index c6818700d..10642eb32 100644 --- a/tests/integration/failover.tcl +++ b/tests/integration/failover.tcl @@ -83,7 +83,11 @@ start_server {} { } else { fail "Failover from node 0 to node 1 did not finish" } + + # stop the write load and make sure no more commands processed stop_write_load $load_handler + wait_load_handlers_disconnected + $node_2 replicaof $node_1_host $node_1_port wait_for_sync $node_0 wait_for_sync $node_2 diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 8715ae999..e4ac83e12 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -21,15 +21,9 @@ start_server {tags {"repl network"}} { stop_bg_complex_data $load_handle0 stop_bg_complex_data $load_handle1 stop_bg_complex_data $load_handle2 - set retry 10 - while {$retry && ([$master debug digest] ne [$slave debug digest])}\ - { - after 1000 - incr retry -1 - } - assert {[$master dbsize] > 0} - - if {[$master debug digest] ne [$slave debug digest]} { + wait_for_condition 100 100 { + [$master debug digest] == [$slave debug digest] + } else { set csv1 [csvdump r] set csv2 [csvdump {r -1}] set fd [open /tmp/repldump1.txt w] @@ -38,10 +32,9 @@ start_server {tags {"repl network"}} { set fd [open /tmp/repldump2.txt w] puts -nonewline $fd $csv2 close $fd - puts "Master - Replica inconsistency" - puts "Run diff -u against /tmp/repldump*.txt for more info" + fail "Master - Replica inconsistency, Run diff -u against /tmp/repldump*.txt for more info" } - assert_equal [r debug digest] [r -1 debug digest] + assert {[$master dbsize] > 0} } } } diff --git a/tests/integration/replication-psync.tcl b/tests/integration/replication-psync.tcl index 3c98723af..08e21d310 100644 --- a/tests/integration/replication-psync.tcl +++ b/tests/integration/replication-psync.tcl @@ -97,15 +97,9 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reco fail "Slave still not connected after some time" } - set retry 10 - while {$retry && ([$master debug digest] ne [$slave debug digest])}\ - { - after 1000 - incr retry -1 - } - assert {[$master dbsize] > 0} - - if {[$master debug digest] ne [$slave debug digest]} { + wait_for_condition 100 100 { + [$master debug digest] == [$slave debug digest] + } else { set csv1 [csvdump r] set csv2 [csvdump {r -1}] set fd [open /tmp/repldump1.txt w] @@ -114,10 +108,9 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reco set fd [open /tmp/repldump2.txt w] puts -nonewline $fd $csv2 close $fd - puts "Master - Replica inconsistency" - puts "Run diff -u against /tmp/repldump*.txt for more info" + fail "Master - Replica inconsistency, Run diff -u against /tmp/repldump*.txt for more info" } - assert_equal [r debug digest] [r -1 debug digest] + assert {[$master dbsize] > 0} eval $cond } } diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index fcef2e9e5..0e8dbcf1b 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -316,15 +316,12 @@ foreach mdl {no yes} { stop_write_load $load_handle3 stop_write_load $load_handle4 - # Make sure that slaves and master have same - # number of keys - wait_for_condition 500 100 { - [$master dbsize] == [[lindex $slaves 0] dbsize] && - [$master dbsize] == [[lindex $slaves 1] dbsize] && - [$master dbsize] == [[lindex $slaves 2] dbsize] - } else { - fail "Different number of keys between master and replica after too long time." - } + # Make sure no more commands processed + wait_load_handlers_disconnected + + wait_for_ofs_sync $master [lindex $slaves 0] + wait_for_ofs_sync $master [lindex $slaves 1] + wait_for_ofs_sync $master [lindex $slaves 2] # Check digests set digest [$master debug digest] diff --git a/tests/support/util.tcl b/tests/support/util.tcl index b00aa159a..d6717f6e1 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -504,6 +504,14 @@ proc stop_write_load {handle} { catch {exec /bin/kill -9 $handle} } +proc wait_load_handlers_disconnected {{level 0}} { + wait_for_condition 50 100 { + ![string match {*name=LOAD_HANDLER*} [r $level client list]] + } else { + fail "load_handler(s) still connected after too long time." + } +} + proc K { x y } { set x } # Shuffle a list with Fisher-Yates algorithm. diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 1a686a2fa..5bdf87256 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -41,15 +41,8 @@ start_server {tags {"aofrw"}} { stop_write_load $load_handle3 stop_write_load $load_handle4 - # Make sure that we remain the only connected client. - # This step is needed to make sure there are no pending writes - # that will be processed between the two "debug digest" calls. - wait_for_condition 50 100 { - [llength [split [string trim [r client list]] "\n"]] == 1 - } else { - puts [r client list] - fail "Clients generating loads are not disconnecting" - } + # Make sure no more commands processed, before taking debug digest + wait_load_handlers_disconnected # Get the data set digest set d1 [r debug digest] From 300b06f1f08cf12f31157a6d1e7213c9b6ba1a13 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Sun, 6 Jun 2021 20:09:06 +0800 Subject: [PATCH 19/79] XTRIM call streamParseAddOrTrimArgsOrReply use wrong arg xadd. (#9047) xtrimCommand call streamParseAddOrTrimArgsOrReply should use xadd==0. When the syntax is valid, it does not cause any bugs because the params of XADD is superset of XTRIM. Just XTRIM will not respond with error on invalid syntax. The syntax of XADD will also be accpeted by XTRIM. (cherry picked from commit 91f3689bf5dc4ce3cf00d9d957b9677b362a205e) --- src/t_stream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_stream.c b/src/t_stream.c index 1b2fe3262..c7be94d6b 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3233,7 +3233,7 @@ void xtrimCommand(client *c) { /* Argument parsing. */ streamAddTrimArgs parsed_args; - if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1) < 0) + if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 0) < 0) return; /* streamParseAddOrTrimArgsOrReply already replied. */ /* If the key does not exist, we are ok returning zero, that is, the From 5a963f360f4b2dbc68d7823d6c42c90e7f91ae9a Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Mon, 7 Jun 2021 19:43:36 +0800 Subject: [PATCH 20/79] Fix XTRIM or XADD with LIMIT may delete more entries than Count. (#9048) The decision to stop trimming due to LIMIT in XADD and XTRIM was after the limit was reached. i.e. the code was deleting **at least** that count of records (from the LIMIT argument's perspective, not the MAXLEN), instead of **up to** that count of records. see #9046 (cherry picked from commit eaa7a7bb93c1ac6dbf347e1c29ac719a12a75158) --- src/t_stream.c | 8 ++++---- tests/unit/type/stream.tcl | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index c7be94d6b..2c30faa06 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -702,16 +702,16 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { int64_t deleted = 0; while (raxNext(&ri)) { - /* Check if we exceeded the amount of work we could do */ - if (limit && deleted >= limit) - break; - if (trim_strategy == TRIM_STRATEGY_MAXLEN && s->length <= maxlen) break; unsigned char *lp = ri.data, *p = lpFirst(lp); int64_t entries = lpGetInteger(p); + /* Check if we exceeded the amount of work we could do */ + if (limit && (deleted + entries) > limit) + break; + /* Check if we can remove the whole node. */ int remove_node; streamID master_id = {0}; /* For MINID */ diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index a89a65299..f1ee56a8a 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -199,6 +199,15 @@ start_server { assert {[r EXISTS otherstream] == 0} } + test {XADD with LIMIT delete entries no more than limit} { + r del yourstream + for {set j 0} {$j < 3} {incr j} { + r XADD yourstream * xitem v + } + r XADD yourstream MAXLEN ~ 0 limit 1 * xitem v + assert {[r XLEN yourstream] == 4} + } + test {XRANGE COUNT works as expected} { assert {[llength [r xrange mystream - + COUNT 10]] == 10} } @@ -525,6 +534,16 @@ start_server { } assert_error ERR* {r XTRIM mystream MAXLEN 1 LIMIT 30} } + + test {XTRIM with LIMIT delete entries no more than limit} { + r del mystream + r config set stream-node-max-entries 2 + for {set j 0} {$j < 3} {incr j} { + r XADD mystream * xitem v + } + assert {[r XTRIM mystream MAXLEN ~ 0 LIMIT 1] == 0} + assert {[r XTRIM mystream MAXLEN ~ 0 LIMIT 2] == 2} + } } start_server {tags {"stream"} overrides {appendonly yes}} { From 288ff19198b01938890f811d4451c8c188ee824a Mon Sep 17 00:00:00 2001 From: guybe7 Date: Tue, 8 Jun 2021 12:36:05 +0300 Subject: [PATCH 21/79] Module API for current command name (#8792) sometimes you can be very deep in the call stack, without access to argv. once you're there you may want your reply/log to contain the command name. (cherry picked from commit e16d3eb998f7017c95ef17179de77aa5b6f2e272) --- src/module.c | 9 +++++++++ src/redismodule.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/module.c b/src/module.c index c76241bfb..df28bc8cc 100644 --- a/src/module.c +++ b/src/module.c @@ -9006,6 +9006,14 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, return res; } +/* Return the name of the command currently running */ +const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) { + if (!ctx || !ctx->client || !ctx->client->cmd) + return NULL; + + return (const char*)ctx->client->cmd->name; +} + /* -------------------------------------------------------------------------- * ## Defrag API * -------------------------------------------------------------------------- */ @@ -9467,6 +9475,7 @@ void moduleRegisterCoreAPI(void) { REGISTER_API(GetServerVersion); REGISTER_API(GetClientCertificate); REGISTER_API(GetCommandKeys); + REGISTER_API(GetCurrentCommandName); REGISTER_API(GetTypeMethodVersion); REGISTER_API(RegisterDefragFunc); REGISTER_API(DefragAlloc); diff --git a/src/redismodule.h b/src/redismodule.h index 5520ca3cc..48a3a9df1 100644 --- a/src/redismodule.h +++ b/src/redismodule.h @@ -836,6 +836,7 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ct REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR; REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR; +REDISMODULE_API const char *(*RedisModule_GetCurrentCommandName)(RedisModuleCtx *ctx) REDISMODULE_ATTR; REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR; REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR; REDISMODULE_API RedisModuleString *(*RedisModule_DefragRedisModuleString)(RedisModuleDefragCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR; @@ -1108,6 +1109,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int REDISMODULE_GET_API(AuthenticateClientWithUser); REDISMODULE_GET_API(GetClientCertificate); REDISMODULE_GET_API(GetCommandKeys); + REDISMODULE_GET_API(GetCurrentCommandName); REDISMODULE_GET_API(RegisterDefragFunc); REDISMODULE_GET_API(DefragAlloc); REDISMODULE_GET_API(DefragRedisModuleString); From fde58863d5e9c6b4431216ffe78c7902608a6513 Mon Sep 17 00:00:00 2001 From: yvette903 <49490087+yvette903@users.noreply.github.com> Date: Tue, 15 Jun 2021 14:21:52 +0800 Subject: [PATCH 22/79] Fix coredump after Client Unpause command when threaded I/O is enabled (#9041) Fix crash when using io-threads-do-reads and issuing CLIENT PAUSE and CLIENT UNPAUSE. This issue was introduced in redis 6.2 together with the FAILOVER command. (cherry picked from commit 096c5fd5d22caa2c21a4863cbaaf64fd8e7107d2) --- src/networking.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 9de105982..d59e49592 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3654,7 +3654,7 @@ int postponeClientRead(client *c) { if (server.io_threads_active && server.io_threads_do_reads && !ProcessingEventsWhileBlocked && - !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ))) + !(c->flags & (CLIENT_MASTER|CLIENT_SLAVE|CLIENT_PENDING_READ|CLIENT_BLOCKED))) { c->flags |= CLIENT_PENDING_READ; listAddNodeHead(server.clients_pending_read,c); @@ -3718,6 +3718,7 @@ int handleClientsWithPendingReadsUsingThreads(void) { c->flags &= ~CLIENT_PENDING_READ; listDelNode(server.clients_pending_read,ln); + serverAssert(!(c->flags & CLIENT_BLOCKED)); if (processPendingCommandsAndResetClient(c) == C_ERR) { /* If the client is no longer valid, we avoid * processing the client later. So we just go From 21a768990c2a68bc1c78d07262aa37140a9232c9 Mon Sep 17 00:00:00 2001 From: Evan Date: Tue, 22 Jun 2021 05:26:48 -0400 Subject: [PATCH 23/79] modules: Add newlen == 0 handling to RM_StringTruncate (#3717) (#3718) Previously, passing 0 for newlen would not truncate the string at all. This adds handling of this case, freeing the old string and creating a new empty string. Other changes: - Move `src/modules/testmodule.c` to `tests/modules/basics.c` - Introduce that basic test into the test suite - Add tests to cover StringTruncate - Add `test-modules` build target for the main makefile - Extend `distclean` build target to clean modules too (cherry picked from commit 1ccf2ca2f475a161b8ca0c574d4c0e6ef9ecf754) --- runtest-moduleapi | 1 + src/Makefile | 5 ++ src/module.c | 21 +++-- tests/modules/Makefile | 1 + .../testmodule.c => tests/modules/basics.c | 85 +++++++++++++++++-- tests/unit/moduleapi/basics.tcl | 13 +++ 6 files changed, 110 insertions(+), 16 deletions(-) rename src/modules/testmodule.c => tests/modules/basics.c (83%) create mode 100644 tests/unit/moduleapi/basics.tcl diff --git a/runtest-moduleapi b/runtest-moduleapi index 7c17501e0..56d149609 100755 --- a/runtest-moduleapi +++ b/runtest-moduleapi @@ -16,6 +16,7 @@ fi $MAKE -C tests/modules && \ $TCLSH tests/test_helper.tcl \ --single unit/moduleapi/commandfilter \ +--single unit/moduleapi/basics \ --single unit/moduleapi/fork \ --single unit/moduleapi/testrdb \ --single unit/moduleapi/infotest \ diff --git a/src/Makefile b/src/Makefile index 62e37cb48..cf3e8c031 100644 --- a/src/Makefile +++ b/src/Makefile @@ -375,6 +375,8 @@ clean: distclean: clean -(cd ../deps && $(MAKE) distclean) + -(cd modules && $(MAKE) clean) + -(cd ../tests/modules && $(MAKE) clean) -(rm -f .make-*) .PHONY: distclean @@ -382,6 +384,9 @@ distclean: clean test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) @(cd ..; ./runtest) +test-modules: $(REDIS_SERVER_NAME) + @(cd ..; ./runtest-moduleapi) + test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) @(cd ..; ./runtest-sentinel) diff --git a/src/module.c b/src/module.c index df28bc8cc..720befe2c 100644 --- a/src/module.c +++ b/src/module.c @@ -2538,14 +2538,19 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { } else { /* Unshare and resize. */ key->value = dbUnshareStringValue(key->db, key->key, key->value); - size_t curlen = sdslen(key->value->ptr); - if (newlen > curlen) { - key->value->ptr = sdsgrowzero(key->value->ptr,newlen); - } else if (newlen < curlen) { - sdsrange(key->value->ptr,0,newlen-1); - /* If the string is too wasteful, reallocate it. */ - if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) - key->value->ptr = sdsRemoveFreeSpace(key->value->ptr); + if (newlen == 0) { + sdsfree(key->value->ptr); + key->value->ptr = sdsempty(); + } else { + size_t curlen = sdslen(key->value->ptr); + if (newlen > curlen) { + key->value->ptr = sdsgrowzero(key->value->ptr,newlen); + } else if (newlen < curlen) { + sdsrange(key->value->ptr,0,newlen-1); + /* If the string is too wasteful, reallocate it. */ + if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) + key->value->ptr = sdsRemoveFreeSpace(key->value->ptr); + } } } return REDISMODULE_OK; diff --git a/tests/modules/Makefile b/tests/modules/Makefile index f56313964..ae611de86 100644 --- a/tests/modules/Makefile +++ b/tests/modules/Makefile @@ -18,6 +18,7 @@ endif TEST_MODULES = \ commandfilter.so \ + basics.so \ testrdb.so \ fork.so \ infotest.so \ diff --git a/src/modules/testmodule.c b/tests/modules/basics.c similarity index 83% rename from src/modules/testmodule.c rename to tests/modules/basics.c index 078c02c5c..9786be1b9 100644 --- a/src/modules/testmodule.c +++ b/tests/modules/basics.c @@ -31,7 +31,7 @@ */ #define REDISMODULE_EXPERIMENTAL_API -#include "../redismodule.h" +#include "redismodule.h" #include /* --------------------------------- Helpers -------------------------------- */ @@ -152,7 +152,58 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { return failTest(ctx, "Could not verify key to be unlinked"); } return RedisModule_ReplyWithSimpleString(ctx, "OK"); +} +/* TEST.STRING.TRUNCATE -- Test truncating an existing string object. */ +int TestStringTruncate(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + REDISMODULE_NOT_USED(argv); + REDISMODULE_NOT_USED(argc); + + RedisModule_Call(ctx, "SET", "cc", "foo", "abcde"); + RedisModuleKey *k = RedisModule_OpenKey(ctx, RedisModule_CreateStringPrintf(ctx, "foo"), REDISMODULE_READ | REDISMODULE_WRITE); + if (!k) return failTest(ctx, "Could not create key"); + + size_t len = 0; + char* s; + + /* expand from 5 to 8 and check null pad */ + if (REDISMODULE_ERR == RedisModule_StringTruncate(k, 8)) { + return failTest(ctx, "Could not truncate string value (8)"); + } + s = RedisModule_StringDMA(k, &len, REDISMODULE_READ); + if (!s) { + return failTest(ctx, "Failed to read truncated string (8)"); + } else if (len != 8) { + return failTest(ctx, "Failed to expand string value (8)"); + } else if (0 != strncmp(s, "abcde\0\0\0", 8)) { + return failTest(ctx, "Failed to null pad string value (8)"); + } + + /* shrink from 8 to 4 */ + if (REDISMODULE_ERR == RedisModule_StringTruncate(k, 4)) { + return failTest(ctx, "Could not truncate string value (4)"); + } + s = RedisModule_StringDMA(k, &len, REDISMODULE_READ); + if (!s) { + return failTest(ctx, "Failed to read truncated string (4)"); + } else if (len != 4) { + return failTest(ctx, "Failed to shrink string value (4)"); + } else if (0 != strncmp(s, "abcd", 4)) { + return failTest(ctx, "Failed to truncate string value (4)"); + } + + /* shrink to 0 */ + if (REDISMODULE_ERR == RedisModule_StringTruncate(k, 0)) { + return failTest(ctx, "Could not truncate string value (0)"); + } + s = RedisModule_StringDMA(k, &len, REDISMODULE_READ); + if (!s) { + return failTest(ctx, "Failed to read truncated string (0)"); + } else if (len != 0) { + return failTest(ctx, "Failed to shrink string value to (0)"); + } + + return RedisModule_ReplyWithSimpleString(ctx, "OK"); } int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, @@ -324,7 +375,11 @@ end: int TestAssertStringReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply, char *str, size_t len) { RedisModuleString *mystr, *expected; - if (RedisModule_CallReplyType(reply) != REDISMODULE_REPLY_STRING) { + if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ERROR) { + RedisModule_Log(ctx,"warning","Test error reply: %s", + RedisModule_CallReplyStringPtr(reply, NULL)); + return 0; + } else if (RedisModule_CallReplyType(reply) != REDISMODULE_REPLY_STRING) { RedisModule_Log(ctx,"warning","Unexpected reply type %d", RedisModule_CallReplyType(reply)); return 0; @@ -345,7 +400,11 @@ int TestAssertStringReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply, char /* Return 1 if the reply matches the specified integer, otherwise log errors * in the server log and return 0. */ int TestAssertIntegerReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply, long long expected) { - if (RedisModule_CallReplyType(reply) != REDISMODULE_REPLY_INTEGER) { + if (RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ERROR) { + RedisModule_Log(ctx,"warning","Test error reply: %s", + RedisModule_CallReplyStringPtr(reply, NULL)); + return 0; + } else if (RedisModule_CallReplyType(reply) != REDISMODULE_REPLY_INTEGER) { RedisModule_Log(ctx,"warning","Unexpected reply type %d", RedisModule_CallReplyType(reply)); return 0; @@ -366,8 +425,11 @@ int TestAssertIntegerReply(RedisModuleCtx *ctx, RedisModuleCallReply *reply, lon reply = RedisModule_Call(ctx,name,__VA_ARGS__); \ } while (0) -/* TEST.IT -- Run all the tests. */ -int TestIt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { +/* TEST.BASICS -- Run all the tests. + * Note: it is useful to run these tests from the module rather than TCL + * since it's easier to check the reply types like that (make a distinction + * between 0 and "0", etc. */ +int TestBasics(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -390,6 +452,9 @@ int TestIt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { T("test.string.append",""); if (!TestAssertStringReply(ctx,reply,"foobar",6)) goto fail; + T("test.string.truncate",""); + if (!TestAssertStringReply(ctx,reply,"OK",2)) goto fail; + T("test.unlink",""); if (!TestAssertStringReply(ctx,reply,"OK",2)) goto fail; @@ -407,7 +472,7 @@ int TestIt(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { fail: RedisModule_ReplyWithSimpleString(ctx, - "SOME TEST NOT PASSED! Check server logs"); + "SOME TEST DID NOT PASS! Check server logs"); return REDISMODULE_OK; } @@ -430,6 +495,10 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) TestStringAppendAM,"write deny-oom",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.string.truncate", + TestStringTruncate,"write deny-oom",1,1,1) == REDISMODULE_ERR) + return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx,"test.string.printf", TestStringPrintf,"write deny-oom",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; @@ -442,8 +511,8 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) TestUnlink,"write deny-oom",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; - if (RedisModule_CreateCommand(ctx,"test.it", - TestIt,"readonly",1,1,1) == REDISMODULE_ERR) + if (RedisModule_CreateCommand(ctx,"test.basics", + TestBasics,"readonly",1,1,1) == REDISMODULE_ERR) return REDISMODULE_ERR; RedisModule_SubscribeToKeyspaceEvents(ctx, diff --git a/tests/unit/moduleapi/basics.tcl b/tests/unit/moduleapi/basics.tcl new file mode 100644 index 000000000..513ace6b9 --- /dev/null +++ b/tests/unit/moduleapi/basics.tcl @@ -0,0 +1,13 @@ +set testmodule [file normalize tests/modules/basics.so] + + +# TEST.CTXFLAGS requires RDB to be disabled, so override save file +start_server {tags {"modules"} overrides {save ""}} { + r module load $testmodule + + test {test module api basics} { + r test.basics + } {ALL TESTS PASSED} + + r module unload test +} From 0be66e4ebe333d4a6d35e2a7490201f792fd45c6 Mon Sep 17 00:00:00 2001 From: Binbin Date: Tue, 29 Jun 2021 15:14:28 +0800 Subject: [PATCH 24/79] ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162) Return a bad score when used with negative count (or count of 1), and non-ziplist encoded zset. Also add test to validate the return value and cover the issue. (cherry picked from commit 4bc5a8324d3cb23ed2cc8a9cd19444a893a6d52c) --- src/t_zset.c | 2 +- tests/unit/type/zset.tcl | 45 ++++++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index fb402816d..400f22072 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -4023,7 +4023,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) { addReplyArrayLen(c,2); addReplyBulkCBuffer(c, key, sdslen(key)); if (withscores) - addReplyDouble(c, dictGetDoubleVal(de)); + addReplyDouble(c, *(double*)dictGetVal(de)); } } else if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) { ziplistEntry *keys, *vals = NULL; diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 96647f778..6008bf5ba 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1616,6 +1616,20 @@ start_server {tags {"zset"}} { return $res } + # Check whether the zset members belong to the zset + proc check_member {mydict res} { + foreach ele $res { + assert {[dict exists $mydict $ele]} + } + } + + # Check whether the zset members and score belong to the zset + proc check_member_and_score {mydict res} { + foreach {key val} $res { + assert_equal $val [dict get $mydict $key] + } + } + foreach {type contents} "ziplist {1 a 2 b 3 c} skiplist {1 a 2 b 3 [randstring 70 90 alpha]}" { set original_max_value [lindex [r config get zset-max-ziplist-value] 1] r config set zset-max-ziplist-value 10 @@ -1676,25 +1690,29 @@ start_server {tags {"zset"}} { # PATH 1: Use negative count. # 1) Check that it returns repeated elements with and without values. + # 2) Check that all the elements actually belong to the original zset. set res [r zrandmember myzset -20] assert_equal [llength $res] 20 + check_member $mydict $res + set res [r zrandmember myzset -1001] assert_equal [llength $res] 1001 + check_member $mydict $res + # again with WITHSCORES set res [r zrandmember myzset -20 withscores] assert_equal [llength $res] 40 + check_member_and_score $mydict $res + set res [r zrandmember myzset -1001 withscores] assert_equal [llength $res] 2002 + check_member_and_score $mydict $res # Test random uniform distribution # df = 9, 40 means 0.00001 probability set res [r zrandmember myzset -1000] assert_lessthan [chi_square_value $res] 40 - - # 2) Check that all the elements actually belong to the original zset. - foreach {key val} $res { - assert {[dict exists $mydict $key]} - } + check_member $mydict $res # 3) Check that eventually all the elements are returned. # Use both WITHSCORES and without @@ -1710,7 +1728,7 @@ start_server {tags {"zset"}} { } else { set res [r zrandmember myzset -3] foreach key $res { - dict append auxset $key $val + dict append auxset $key } } if {[lsort [dict keys $mydict]] eq @@ -1726,11 +1744,13 @@ start_server {tags {"zset"}} { set res [r zrandmember myzset $size] assert_equal [llength $res] 10 assert_equal [lsort $res] [lsort [dict keys $mydict]] + check_member $mydict $res # again with WITHSCORES set res [r zrandmember myzset $size withscores] assert_equal [llength $res] 20 assert_equal [lsort $res] [lsort $mydict] + check_member_and_score $mydict $res } # PATH 3: Ask almost as elements as there are in the set. @@ -1742,18 +1762,17 @@ start_server {tags {"zset"}} { # # We can test both the code paths just changing the size but # using the same code. - foreach size {8 2} { + foreach size {1 2 8} { + # 1) Check that all the elements actually belong to the + # original set. set res [r zrandmember myzset $size] assert_equal [llength $res] $size + check_member $mydict $res + # again with WITHSCORES set res [r zrandmember myzset $size withscores] assert_equal [llength $res] [expr {$size * 2}] - - # 1) Check that all the elements actually belong to the - # original set. - foreach ele [dict keys $res] { - assert {[dict exists $mydict $ele]} - } + check_member_and_score $mydict $res # 2) Check that eventually all the elements are returned. # Use both WITHSCORES and without From 3b32512dc9239f5624e77d7bec525e97dbef1d55 Mon Sep 17 00:00:00 2001 From: guybe7 Date: Tue, 29 Jun 2021 13:34:18 +0200 Subject: [PATCH 25/79] Include sizeof(struct stream) in objectComputeSize (#9164) Affects MEMORY USAGE (cherry picked from commit 4434cebbb38ae2ef39b2d74ed3e6cd8d1a061d0c) --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index c7b25ffd4..c6381a231 100644 --- a/src/object.c +++ b/src/object.c @@ -881,7 +881,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) { } } else if (o->type == OBJ_STREAM) { stream *s = o->ptr; - asize = sizeof(*o); + asize = sizeof(*o)+sizeof(*s); asize += streamRadixTreeMemoryUsage(s->rax); /* Now we have to add the listpacks. The last listpack is often non From adc47482487909a2ee9718cc4de721e441e15d9a Mon Sep 17 00:00:00 2001 From: Leibale Eidelman Date: Tue, 29 Jun 2021 09:38:10 -0400 Subject: [PATCH 26/79] fix ZRANGESTORE - should return 0 when src points to an empty key (#9089) mistakenly it used to return an empty array rather than 0. Co-authored-by: Oran Agra (cherry picked from commit 95274f1f8a3ef4cb4033beecfaa99ea1439ed170) --- src/t_zset.c | 7 ++++++- tests/unit/type/zset.tcl | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 400f22072..7a3394d47 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3663,7 +3663,12 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st lookupKeyWrite(c->db,key) : lookupKeyRead(c->db,key); if (zobj == NULL) { - addReply(c,shared.emptyarray); + if (store) { + handler->beginResultEmission(handler); + handler->finalizeResultEmission(handler, 0); + } else { + addReply(c, shared.emptyarray); + } goto cleanup; } diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 6008bf5ba..554d9c2c1 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1568,6 +1568,19 @@ start_server {tags {"zset"}} { r zrange z1 5 0 BYSCORE REV LIMIT 0 2 WITHSCORES } {d 4 c 3} + test {ZRANGESTORE - src key missing} { + set res [r zrangestore z2{t} missing{t} 0 -1] + assert_equal $res 0 + r exists z2{t} + } {0} + + test {ZRANGESTORE - src key wrong type} { + r zadd z2{t} 1 a + r set foo{t} bar + assert_error "*WRONGTYPE*" {r zrangestore z2{t} foo{t} 0 -1} + r zrange z2{t} 0 -1 + } {a} + test {ZRANGESTORE - empty range} { set res [r zrangestore z2 z1 5 6] assert_equal $res 0 From 2186c2e945a54941a7e4e63ce502db761e18afbf Mon Sep 17 00:00:00 2001 From: Mikhail Fesenko Date: Wed, 30 Jun 2021 16:49:54 +0300 Subject: [PATCH 27/79] redis-cli --rdb: fix broken fsync/ftruncate for stdout (#9135) A change in redis 6.2 caused redis-cli --rdb that's directed to stdout to fail because fsync fails. This commit avoids doing ftruncate (fails with a warning) and fsync (fails with an error) when the output file is `-`, and adds the missing documentation that `-` means stdout. Co-authored-by: Oran Agra Co-authored-by: Wang Yuan (cherry picked from commit 74fe15b3602ed7c003b5c53e45e31f7aa6d4a86f) --- src/redis-cli.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 8e049f186..d4d5e7459 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1896,6 +1896,7 @@ static void usage(void) { " --lru-test Simulate a cache workload with an 80-20 distribution.\n" " --replica Simulate a replica showing commands received from the master.\n" " --rdb Transfer an RDB dump from remote server to local file.\n" +" Use filename of \"-\" to write to stdout.\n" " --pipe Transfer raw Redis protocol from stdin to server.\n" " --pipe-timeout In --pipe mode, abort with error if after sending all data.\n" " no reply is received within seconds.\n" @@ -7154,8 +7155,9 @@ static void getRDB(clusterManagerNode *node) { payload, filename); } + int write_to_stdout = !strcmp(filename,"-"); /* Write to file. */ - if (!strcmp(filename,"-")) { + if (write_to_stdout) { fd = STDOUT_FILENO; } else { fd = open(filename, O_CREAT|O_WRONLY, 0644); @@ -7197,7 +7199,7 @@ static void getRDB(clusterManagerNode *node) { } if (usemark) { payload = ULLONG_MAX - payload - RDB_EOF_MARK_SIZE; - if (ftruncate(fd, payload) == -1) + if (!write_to_stdout && ftruncate(fd, payload) == -1) fprintf(stderr,"ftruncate failed: %s.\n", strerror(errno)); fprintf(stderr,"Transfer finished with success after %llu bytes\n", payload); } else { @@ -7206,7 +7208,7 @@ static void getRDB(clusterManagerNode *node) { redisFree(s); /* Close the connection ASAP as fsync() may take time. */ if (node) node->context = NULL; - if (fsync(fd) == -1) { + if (!write_to_stdout && fsync(fd) == -1) { fprintf(stderr,"Fail to fsync '%s': %s\n", filename, strerror(errno)); exit(1); } From af5bd089a089253f2f1cd683ce381cd2a6c95e69 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 4 Jul 2021 19:43:58 +0300 Subject: [PATCH 28/79] Tests: add a way to read raw RESP protocol reponses (#9193) This makes it possible to distinguish between null response and an empty array (currently the tests infra translates both to an empty string/list) (cherry picked from commit 7103367ad44b4241e59a709771cb464aa2a86b20) --- tests/support/redis.tcl | 13 ++++++++++++- tests/unit/protocol.tcl | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index 4d321c975..285b53574 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -34,13 +34,14 @@ array set ::redis::fd {} array set ::redis::addr {} array set ::redis::blocking {} array set ::redis::deferred {} +array set ::redis::readraw {} array set ::redis::reconnect {} array set ::redis::tls {} array set ::redis::callback {} array set ::redis::state {} ;# State in non-blocking reply reading array set ::redis::statestack {} ;# Stack of states, for nested mbulks -proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}}} { +proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}} {readraw 0}} { if {$tls} { package require tls ::tls::init \ @@ -58,6 +59,7 @@ proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}}} { set ::redis::addr($id) [list $server $port] set ::redis::blocking($id) 1 set ::redis::deferred($id) $defer + set ::redis::readraw($id) $readraw set ::redis::reconnect($id) 0 set ::redis::tls($id) $tls ::redis::redis_reset_state $id @@ -158,6 +160,7 @@ proc ::redis::__method__close {id fd} { catch {unset ::redis::addr($id)} catch {unset ::redis::blocking($id)} catch {unset ::redis::deferred($id)} + catch {unset ::redis::readraw($id)} catch {unset ::redis::reconnect($id)} catch {unset ::redis::tls($id)} catch {unset ::redis::state($id)} @@ -174,6 +177,10 @@ proc ::redis::__method__deferred {id fd val} { set ::redis::deferred($id) $val } +proc ::redis::__method__readraw {id fd val} { + set ::redis::readraw($id) $val +} + proc ::redis::redis_write {fd buf} { puts -nonewline $fd $buf } @@ -241,6 +248,10 @@ proc ::redis::redis_read_null fd { } proc ::redis::redis_read_reply {id fd} { + if {$::redis::readraw($id)} { + return [redis_read_line $fd] + } + set type [read $fd 1] switch -exact -- $type { _ {redis_read_null $fd} diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index 442c23de6..ffc751f8f 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -102,6 +102,40 @@ start_server {tags {"protocol network"}} { } {*Protocol error*} } unset c + + # recover the broken connection + reconnect + r ping + + # raw RESP response tests + r readraw 1 + + test "raw protocol response" { + r srandmember nonexisting_key + } {*-1} + + r deferred 1 + + test "raw protocol response - deferred" { + r srandmember nonexisting_key + r read + } {*-1} + + test "raw protocol response - multiline" { + r sadd ss a + assert_equal [r read] {:1} + r srandmember ss 100 + assert_equal [r read] {*1} + assert_equal [r read] {$1} + assert_equal [r read] {a} + } + + # restore connection settings + r readraw 0 + r deferred 0 + + # check the connection still works + assert_equal [r ping] {PONG} } start_server {tags {"regression"}} { From 519955ecf26b8043de575b5660d937968d1670f0 Mon Sep 17 00:00:00 2001 From: Binbin Date: Mon, 5 Jul 2021 15:41:57 +0800 Subject: [PATCH 29/79] hrandfield and zrandmember with count should return emptyarray when key does not exist. (#9178) due to a copy-paste bug, it used to reply with null response rather than empty array. this commit includes new tests that are looking at the RESP response directly in order to be able to tell the difference between them. Co-authored-by: Oran Agra (cherry picked from commit a418a2d3fc0250c094802d7e8ea64d96eedfda07) --- src/t_hash.c | 4 ++-- src/t_zset.c | 4 ++-- tests/unit/type/hash.tcl | 13 +++++++++++++ tests/unit/type/set.tcl | 17 +++++++++++++++++ tests/unit/type/zset.tcl | 13 +++++++++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index d88b80b0f..ea0606fb0 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -986,7 +986,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { int uniq = 1; robj *hash; - if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) + if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray)) == NULL || checkType(c,hash,OBJ_HASH)) return; size = hashTypeLength(hash); @@ -1175,7 +1175,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) { } } -/* HRANDFIELD [ WITHVALUES] */ +/* HRANDFIELD key [ [WITHVALUES]] */ void hrandfieldCommand(client *c) { long l; int withvalues = 0; diff --git a/src/t_zset.c b/src/t_zset.c index 7a3394d47..c501518a3 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3992,7 +3992,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) { int uniq = 1; robj *zsetobj; - if ((zsetobj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp])) + if ((zsetobj = lookupKeyReadOrReply(c, c->argv[1], shared.emptyarray)) == NULL || checkType(c, zsetobj, OBJ_ZSET)) return; size = zsetLength(zsetobj); @@ -4177,7 +4177,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) { } } -/* ZRANDMEMBER [ WITHSCORES] */ +/* ZRANDMEMBER key [ [WITHSCORES]] */ void zrandmemberCommand(client *c) { long l; int withscores = 0; diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index fcf97eed7..f2a503722 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -72,6 +72,19 @@ start_server {tags {"hash"}} { r hrandfield nonexisting_key 100 } {} + # Make sure we can distinguish between an empty array and a null response + r readraw 1 + + test "HRANDFIELD count of 0 is handled correctly - emptyarray" { + r hrandfield myhash 0 + } {*0} + + test "HRANDFIELD with against non existing key - emptyarray" { + r hrandfield nonexisting_key 100 + } {*0} + + r readraw 0 + foreach {type contents} " hashtable {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {[randstring 70 90 alpha] 10}} ziplist {{a 1} {b 2} {c 3} {d 4} {e 5} {6 f} {7 g} {8 h} {9 i} {10 j}} " { diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index 5548ca3a2..c5a4c687f 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -403,10 +403,27 @@ start_server { assert {[lsort $union] eq [lsort $content]} } + test "SRANDMEMBER count of 0 is handled correctly" { + r srandmember myset 0 + } {} + test "SRANDMEMBER with against non existing key" { r srandmember nonexisting_key 100 } {} + # Make sure we can distinguish between an empty array and a null response + r readraw 1 + + test "SRANDMEMBER count of 0 is handled correctly - emptyarray" { + r srandmember myset 0 + } {*0} + + test "SRANDMEMBER with against non existing key - emptyarray" { + r srandmember nonexisting_key 100 + } {*0} + + r readraw 0 + foreach {type contents} { hashtable { 1 5 10 50 125 50000 33959417 4775547 65434162 diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 554d9c2c1..004eaf8a3 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1681,6 +1681,19 @@ start_server {tags {"zset"}} { r zrandmember nonexisting_key 100 } {} + # Make sure we can distinguish between an empty array and a null response + r readraw 1 + + test "ZRANDMEMBER count of 0 is handled correctly - emptyarray" { + r zrandmember myzset 0 + } {*0} + + test "ZRANDMEMBER with against non existing key - emptyarray" { + r zrandmember nonexisting_key 100 + } {*0} + + r readraw 0 + foreach {type contents} " skiplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 [randstring 70 90 alpha]} ziplist {1 a 2 b 3 c 4 d 5 e 6 f 7 g 7 h 9 i 10 j} " { From 8149476f66d6e1e2531e622fd35e81cf3c0cd87c Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 14 Jul 2021 19:14:31 +0300 Subject: [PATCH 30/79] Test infra, handle RESP3 attributes and big-numbers and bools (#9235) - promote the code in DEBUG PROTOCOL to addReplyBigNum - DEBUG PROTOCOL ATTRIB skips the attribute when client is RESP2 - networking.c addReply for push and attributes generate assertion when called on a RESP2 client, anything else would produce a broken protocol that clients can't handle. (cherry picked from commit 6a5bac309e868deef749c36949723b415de2496f) --- src/debug.c | 14 ++++++----- src/networking.c | 28 +++++++++++++-------- src/server.h | 1 + tests/support/redis.tcl | 52 +++++++++++++++++++++++++-------------- tests/unit/protocol.tcl | 54 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 34 deletions(-) diff --git a/src/debug.c b/src/debug.c index 098ce6ef7..f521490a7 100644 --- a/src/debug.c +++ b/src/debug.c @@ -721,7 +721,7 @@ NULL } else if (!strcasecmp(name,"double")) { addReplyDouble(c,3.14159265359); } else if (!strcasecmp(name,"bignum")) { - addReplyProto(c,"(1234567999999999999999999999999999999\r\n",40); + addReplyBigNum(c,"1234567999999999999999999999999999999",37); } else if (!strcasecmp(name,"null")) { addReplyNull(c); } else if (!strcasecmp(name,"array")) { @@ -737,11 +737,13 @@ NULL addReplyBool(c, j == 1); } } else if (!strcasecmp(name,"attrib")) { - addReplyAttributeLen(c,1); - addReplyBulkCString(c,"key-popularity"); - addReplyArrayLen(c,2); - addReplyBulkCString(c,"key:123"); - addReplyLongLong(c,90); + if (c->resp >= 3) { + addReplyAttributeLen(c,1); + addReplyBulkCString(c,"key-popularity"); + addReplyArrayLen(c,2); + addReplyBulkCString(c,"key:123"); + addReplyLongLong(c,90); + } /* Attributes are not real replies, so a well formed reply should * also have a normal reply type after the attribute. */ addReplyBulkCString(c,"Some real reply following the attribute"); diff --git a/src/networking.c b/src/networking.c index d59e49592..51c58ff20 100644 --- a/src/networking.c +++ b/src/networking.c @@ -649,14 +649,13 @@ void setDeferredSetLen(client *c, void *node, long length) { } void setDeferredAttributeLen(client *c, void *node, long length) { - int prefix = c->resp == 2 ? '*' : '|'; - if (c->resp == 2) length *= 2; - setDeferredAggregateLen(c,node,length,prefix); + serverAssert(c->resp >= 3); + setDeferredAggregateLen(c,node,length,'|'); } void setDeferredPushLen(client *c, void *node, long length) { - int prefix = c->resp == 2 ? '*' : '>'; - setDeferredAggregateLen(c,node,length,prefix); + serverAssert(c->resp >= 3); + setDeferredAggregateLen(c,node,length,'>'); } /* Add a double as a bulk reply */ @@ -685,6 +684,16 @@ void addReplyDouble(client *c, double d) { } } +void addReplyBigNum(client *c, const char* num, size_t len) { + if (c->resp == 2) { + addReplyBulkCBuffer(c, num, len); + } else { + addReplyProto(c,"(",1); + addReplyProto(c,num,len); + addReply(c,shared.crlf); + } +} + /* Add a long double as a bulk reply, but uses a human readable formatting * of the double instead of exposing the crude behavior of doubles to the * dear user. */ @@ -756,14 +765,13 @@ void addReplySetLen(client *c, long length) { } void addReplyAttributeLen(client *c, long length) { - int prefix = c->resp == 2 ? '*' : '|'; - if (c->resp == 2) length *= 2; - addReplyAggregateLen(c,length,prefix); + serverAssert(c->resp >= 3); + addReplyAggregateLen(c,length,'|'); } void addReplyPushLen(client *c, long length) { - int prefix = c->resp == 2 ? '*' : '>'; - addReplyAggregateLen(c,length,prefix); + serverAssert(c->resp >= 3); + addReplyAggregateLen(c,length,'>'); } void addReplyNull(client *c) { diff --git a/src/server.h b/src/server.h index 64d08db60..f57a720e9 100644 --- a/src/server.h +++ b/src/server.h @@ -1840,6 +1840,7 @@ void addReplyErrorSds(client *c, sds err); void addReplyError(client *c, const char *err); void addReplyStatus(client *c, const char *status); void addReplyDouble(client *c, double d); +void addReplyBigNum(client *c, const char* num, size_t len); void addReplyHumanLongDouble(client *c, long double d); void addReplyLongLong(client *c, long long ll); void addReplyArrayLen(client *c, long length); diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index 285b53574..978163e98 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -247,30 +247,46 @@ proc ::redis::redis_read_null fd { return {} } +proc ::redis::redis_read_bool fd { + set v [redis_read_line $fd] + if {$v == "t"} {return 1} + if {$v == "f"} {return 0} + return -code error "Bad protocol, '$v' as bool type" +} + proc ::redis::redis_read_reply {id fd} { if {$::redis::readraw($id)} { return [redis_read_line $fd] } - set type [read $fd 1] - switch -exact -- $type { - _ {redis_read_null $fd} - : - - + {redis_read_line $fd} - , {expr {double([redis_read_line $fd])}} - - {return -code error [redis_read_line $fd]} - $ {redis_bulk_read $fd} - > - - ~ - - * {redis_multi_bulk_read $id $fd} - % {redis_read_map $id $fd} - default { - if {$type eq {}} { - catch {close $fd} - set ::redis::fd($id) {} - return -code error "I/O error reading reply" + while {1} { + set type [read $fd 1] + switch -exact -- $type { + _ {return [redis_read_null $fd]} + : - + ( - + + {return [redis_read_line $fd]} + , {return [expr {double([redis_read_line $fd])}]} + # {return [redis_read_bool $fd]} + - {return -code error [redis_read_line $fd]} + $ {return [redis_bulk_read $fd]} + > - + ~ - + * {return [redis_multi_bulk_read $id $fd]} + % {return [redis_read_map $id $fd]} + | { + # ignore attributes for now (nowhere to store them) + redis_read_map $id $fd + continue + } + default { + if {$type eq {}} { + catch {close $fd} + set ::redis::fd($id) {} + return -code error "I/O error reading reply" + } + return -code error "Bad protocol, '$type' as reply type byte" } - return -code error "Bad protocol, '$type' as reply type byte" } } } diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index ffc751f8f..a3d8f7e89 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -136,6 +136,60 @@ start_server {tags {"protocol network"}} { # check the connection still works assert_equal [r ping] {PONG} + + test {RESP3 attributes} { + r hello 3 + set res [r debug protocol attrib] + # currently the parser in redis.tcl ignores the attributes + + # restore state + r hello 2 + set _ $res + } {Some real reply following the attribute} + + test {RESP3 attributes readraw} { + r hello 3 + r readraw 1 + r deferred 1 + + r debug protocol attrib + assert_equal [r read] {|1} + assert_equal [r read] {$14} + assert_equal [r read] {key-popularity} + assert_equal [r read] {*2} + assert_equal [r read] {$7} + assert_equal [r read] {key:123} + assert_equal [r read] {:90} + assert_equal [r read] {$39} + assert_equal [r read] {Some real reply following the attribute} + + # restore state + r readraw 0 + r deferred 0 + r hello 2 + set _ {} + } {} + + test {RESP3 attributes on RESP2} { + r hello 2 + set res [r debug protocol attrib] + set _ $res + } {Some real reply following the attribute} + + test "test big number parsing" { + r hello 3 + r debug protocol bignum + } {1234567999999999999999999999999999999} + + test "test bool parsing" { + r hello 3 + assert_equal [r debug protocol true] 1 + assert_equal [r debug protocol false] 0 + r hello 2 + assert_equal [r debug protocol true] 1 + assert_equal [r debug protocol false] 0 + set _ {} + } {} } start_server {tags {"regression"}} { From 2ac883ecc13c2bf858756e854882abb33f101f90 Mon Sep 17 00:00:00 2001 From: Rob Snyder Date: Wed, 30 Jun 2021 09:46:06 -0400 Subject: [PATCH 31/79] Fix ziplist length updates on bigendian platforms (#2080) Adds call to intrev16ifbe to ensure ZIPLIST_LENGTH is compared correctly (cherry picked from commit eaa52719a355c4467d0383c1c9f5184c9c14fe5a) --- src/ziplist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ziplist.c b/src/ziplist.c index 85cb50991..aae86c1f2 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -263,7 +263,7 @@ * to stay there to signal that a full scan is needed to get the number of * items inside the ziplist. */ #define ZIPLIST_INCR_LENGTH(zl,incr) { \ - if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \ + if (intrev16ifbe(ZIPLIST_LENGTH(zl)) < UINT16_MAX) \ ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \ } From 740c41c0f26f0773a11c7121dc2aff1fdb34f9b1 Mon Sep 17 00:00:00 2001 From: Mikhail Fesenko Date: Wed, 7 Jul 2021 08:26:26 +0300 Subject: [PATCH 32/79] Direct redis-cli repl prints to stderr, because --rdb can print to stdout. fflush stdout after responses (#9136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. redis-cli can output --rdb data to stdout but redis-cli also write some messages to stdout which will mess up the rdb. 2. Make redis-cli flush stdout when printing a reply This was needed in order to fix a hung in redis-cli test that uses --replica. Note that printf does flush when there's a newline, but fwrite does not. 3. fix the redis-cli --replica test which used to pass previously because it didn't really care what it read, and because redis-cli used printf to print these other things to stdout. 4. improve redis-cli --replica test to run with both diskless and disk-based. Co-authored-by: Oran Agra Co-authored-by: Viktor Söderqvist (cherry picked from commit 1eb4baa5b8e76adc337ae9fab49acc2585a0cdd0) --- src/redis-cli.c | 5 +++-- tests/integration/redis-cli.tcl | 28 ++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d4d5e7459..06ccf1563 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1312,6 +1312,7 @@ static int cliReadReply(int output_raw_strings) { if (output) { out = cliFormatReply(reply, config.output, output_raw_strings); fwrite(out,sdslen(out),1,stdout); + fflush(stdout); sdsfree(out); } freeReplyObject(reply); @@ -6990,7 +6991,7 @@ static void latencyDistMode(void) { #define RDB_EOF_MARK_SIZE 40 void sendReplconf(const char* arg1, const char* arg2) { - printf("sending REPLCONF %s %s\n", arg1, arg2); + fprintf(stderr, "sending REPLCONF %s %s\n", arg1, arg2); redisReply *reply = redisCommand(context, "REPLCONF %s %s", arg1, arg2); /* Handle any error conditions */ @@ -7050,7 +7051,7 @@ unsigned long long sendSync(redisContext *c, char *out_eof) { } *p = '\0'; if (buf[0] == '-') { - printf("SYNC with master failed: %s\n", buf); + fprintf(stderr, "SYNC with master failed: %s\n", buf); exit(1); } if (strncmp(buf+1,"EOF:",4) == 0 && strlen(buf+5) >= RDB_EOF_MARK_SIZE) { diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 7e8b41fca..d58229bb4 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -283,9 +283,9 @@ start_server {tags {"cli"}} { assert_equal {key:2} [run_cli --scan --quoted-pattern {"*:\x32"}] } - test "Connecting as a replica" { + proc test_redis_cli_repl {} { set fd [open_cli "--replica"] - wait_for_condition 500 500 { + wait_for_condition 500 100 { [string match {*slave0:*state=online*} [r info]] } else { fail "redis-cli --replica did not connect" @@ -294,14 +294,30 @@ start_server {tags {"cli"}} { for {set i 0} {$i < 100} {incr i} { r set test-key test-value-$i } - r client kill type slave - catch { - assert_match {*SET*key-a*} [read_cli $fd] + + wait_for_condition 500 100 { + [string match {*test-value-99*} [read_cli $fd]] + } else { + fail "redis-cli --replica didn't read commands" } - close_cli $fd + fconfigure $fd -blocking true + r client kill type slave + catch { close_cli $fd } err + assert_match {*Server closed the connection*} $err } + test "Connecting as a replica" { + # Disk-based master + assert_match "OK" [r config set repl-diskless-sync no] + test_redis_cli_repl + + # Disk-less master + assert_match "OK" [r config set repl-diskless-sync yes] + assert_match "OK" [r config set repl-diskless-sync-delay 0] + test_redis_cli_repl + } {} + test "Piping raw protocol" { set cmds [tmpfile "cli_cmds"] set cmds_fd [open $cmds "w"] From f497eb3cca6242f7fd27aca03934332c40cb42a6 Mon Sep 17 00:00:00 2001 From: Jason Elbaum Date: Wed, 16 Jun 2021 09:29:57 +0300 Subject: [PATCH 33/79] Change return value type for ZPOPMAX/MIN in RESP3 (#8981) When using RESP3, ZPOPMAX/ZPOPMIN should return nested arrays for consistency with other commands (e.g. ZRANGE). We do that only when COUNT argument is present (similarly to how LPOP behaves). for reasoning see https://github.com/redis/redis/issues/8824#issuecomment-855427955 This is a breaking change only when RESP3 is used, and COUNT argument is present! (cherry picked from commit 7f342020dcbdf9abe754d6b666efdeded7063870) --- src/t_zset.c | 19 +++++++++++++++---- tests/unit/type/zset.tcl | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index c501518a3..3b9ebd2bd 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -3825,11 +3825,16 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey } void *arraylen_ptr = addReplyDeferredLen(c); - long arraylen = 0; + long result_count = 0; /* We emit the key only for the blocking variant. */ if (emitkey) addReplyBulk(c,key); + /* Respond with a single (flat) array in RESP2 or if countarg is not + * provided (returning a single element). In RESP3, when countarg is + * provided, use nested array. */ + int use_nested_array = c->resp > 2 && countarg != NULL; + /* Remove the element. */ do { if (zobj->encoding == OBJ_ENCODING_ZIPLIST) { @@ -3872,16 +3877,19 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey serverAssertWithInfo(c,zobj,zsetDel(zobj,ele)); server.dirty++; - if (arraylen == 0) { /* Do this only for the first iteration. */ + if (result_count == 0) { /* Do this only for the first iteration. */ char *events[2] = {"zpopmin","zpopmax"}; notifyKeyspaceEvent(NOTIFY_ZSET,events[where],key,c->db->id); signalModifiedKey(c,c->db,key); } + if (use_nested_array) { + addReplyArrayLen(c,2); + } addReplyBulkCBuffer(c,ele,sdslen(ele)); addReplyDouble(c,score); sdsfree(ele); - arraylen += 2; + ++result_count; /* Remove the key, if indeed needed. */ if (zsetLength(zobj) == 0) { @@ -3891,7 +3899,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey } } while(--count); - setDeferredArrayLen(c,arraylen_ptr,arraylen + (emitkey != 0)); + if (!use_nested_array) { + result_count *= 2; + } + setDeferredArrayLen(c,arraylen_ptr,result_count + (emitkey != 0)); } /* ZPOPMIN key [] */ diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 004eaf8a3..94b2ab480 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -960,6 +960,39 @@ start_server {tags {"zset"}} { assert_equal 1 [r zcard z2] } + test "Basic ZPOP - $encoding RESP3" { + r hello 3 + r del z1 + create_zset z1 {0 a 1 b 2 c 3 d} + assert_equal {a 0.0} [r zpopmin z1] + assert_equal {d 3.0} [r zpopmax z1] + r hello 2 + } + + test "ZPOP with count - $encoding RESP3" { + r hello 3 + r del z1 + create_zset z1 {0 a 1 b 2 c 3 d} + assert_equal {{a 0.0} {b 1.0}} [r zpopmin z1 2] + assert_equal {{d 3.0} {c 2.0}} [r zpopmax z1 2] + r hello 2 + } + + test "BZPOP - $encoding RESP3" { + r hello 3 + set rd [redis_deferring_client] + create_zset zset {0 a 1 b 2 c} + + $rd bzpopmin zset 5 + assert_equal {zset a 0} [$rd read] + $rd bzpopmin zset 5 + assert_equal {zset b 1} [$rd read] + $rd bzpopmax zset 5 + assert_equal {zset c 2} [$rd read] + assert_equal 0 [r exists zset] + r hello 2 + } + r config set zset-max-ziplist-entries $original_max_entries r config set zset-max-ziplist-value $original_max_value } From 933a6ca272d7e5e424d37e75fcbc4739e5908993 Mon Sep 17 00:00:00 2001 From: Binbin Date: Sun, 13 Jun 2021 15:53:46 +0800 Subject: [PATCH 34/79] Fix accidental deletion of sinterstore command when we meet wrong type error. (#9032) SINTERSTORE would have deleted the dest key right away, even when later on it is bound to fail on an (WRONGTYPE) error. With this change it first picks up all the input keys, and only later delete the dest key if one is empty. Also add more tests for some commands. Mainly focus on - `wrong type error`: expand test case (base on sinter bug) in non-store variant add tests for store variant (although it exists in non-store variant, i think it would be better to have same tests) - the dstkey result when we meet `non-exist key (empty set)` in *store sdiff: - improve test case about wrong type error (the one we found in sinter, although it is safe in sdiff) - add test about using non-exist key (treat it like an empty set) sdiffstore: - according to sdiff test case, also add some tests about `wrong type error` and `non-exist key` - the different is that in sdiffstore, we will consider the `dstkey` result sunion/sunionstore add more tests (same as above) sinter/sinterstore also same as above ... (cherry picked from commit b8a5da80c49501773f8778aaf5cbf595cef615e4) --- src/t_set.c | 42 ++++++--- tests/unit/type/set.tcl | 195 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 215 insertions(+), 22 deletions(-) diff --git a/src/t_set.c b/src/t_set.c index bf250baa2..d0c54848e 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -858,25 +858,17 @@ void sinterGenericCommand(client *c, robj **setkeys, int64_t intobj; void *replylen = NULL; unsigned long j, cardinality = 0; - int encoding; + int encoding, empty = 0; for (j = 0; j < setnum; j++) { robj *setobj = dstkey ? lookupKeyWrite(c->db,setkeys[j]) : lookupKeyRead(c->db,setkeys[j]); if (!setobj) { - zfree(sets); - if (dstkey) { - if (dbDelete(c->db,dstkey)) { - signalModifiedKey(c,c->db,dstkey); - notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id); - server.dirty++; - } - addReply(c,shared.czero); - } else { - addReply(c,shared.emptyset[c->resp]); - } - return; + /* A NULL is considered an empty set */ + empty += 1; + sets[j] = NULL; + continue; } if (checkType(c,setobj,OBJ_SET)) { zfree(sets); @@ -884,6 +876,24 @@ void sinterGenericCommand(client *c, robj **setkeys, } sets[j] = setobj; } + + /* Set intersection with an empty set always results in an empty set. + * Return ASAP if there is an empty set. */ + if (empty > 0) { + zfree(sets); + if (dstkey) { + if (dbDelete(c->db,dstkey)) { + signalModifiedKey(c,c->db,dstkey); + notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id); + server.dirty++; + } + addReply(c,shared.czero); + } else { + addReply(c,shared.emptyset[c->resp]); + } + return; + } + /* Sort sets from the smallest to largest, this will improve our * algorithm's performance */ qsort(sets,setnum,sizeof(robj*),qsortCompareSetsByCardinality); @@ -977,10 +987,12 @@ void sinterGenericCommand(client *c, robj **setkeys, zfree(sets); } +/* SINTER key [key ...] */ void sinterCommand(client *c) { sinterGenericCommand(c,c->argv+1,c->argc-1,NULL); } +/* SINTERSTORE destination key [key ...] */ void sinterstoreCommand(client *c) { sinterGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]); } @@ -1150,18 +1162,22 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum, zfree(sets); } +/* SUNION key [key ...] */ void sunionCommand(client *c) { sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,SET_OP_UNION); } +/* SUNIONSTORE destination key [key ...] */ void sunionstoreCommand(client *c) { sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],SET_OP_UNION); } +/* SDIFF key [key ...] */ void sdiffCommand(client *c) { sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,SET_OP_DIFF); } +/* SDIFFSTORE destination key [key ...] */ void sdiffstoreCommand(client *c) { sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],SET_OP_DIFF); } diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index c5a4c687f..bc5accb41 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -276,14 +276,86 @@ start_server { } } - test "SINTER against non-set should throw error" { - r set key1 x - assert_error "WRONGTYPE*" {r sinter key1 noset} + test "SDIFF against non-set should throw error" { + # with an empty set + r set key1{t} x + assert_error "WRONGTYPE*" {r sdiff key1{t} noset{t}} + # different order + assert_error "WRONGTYPE*" {r sdiff noset{t} key1{t}} + + # with a legal set + r del set1{t} + r sadd set1{t} a b c + assert_error "WRONGTYPE*" {r sdiff key1{t} set1{t}} + # different order + assert_error "WRONGTYPE*" {r sdiff set1{t} key1{t}} } - test "SUNION against non-set should throw error" { - r set key1 x - assert_error "WRONGTYPE*" {r sunion key1 noset} + test "SDIFF should handle non existing key as empty" { + r del set1{t} set2{t} set3{t} + + r sadd set1{t} a b c + r sadd set2{t} b c d + assert_equal {a} [lsort [r sdiff set1{t} set2{t} set3{t}]] + assert_equal {} [lsort [r sdiff set3{t} set2{t} set1{t}]] + } + + test "SDIFFSTORE against non-set should throw error" { + r del set1{t} set2{t} set3{t} key1{t} + r set key1{t} x + + # with en empty dstkey + assert_error "WRONGTYPE*" {r SDIFFSTORE set3{t} key1{t} noset{t}} + assert_equal 0 [r exists set3{t}] + assert_error "WRONGTYPE*" {r SDIFFSTORE set3{t} noset{t} key1{t}} + assert_equal 0 [r exists set3{t}] + + # with a legal dstkey + r sadd set1{t} a b c + r sadd set2{t} b c d + r sadd set3{t} e + assert_error "WRONGTYPE*" {r SDIFFSTORE set3{t} key1{t} set1{t} noset{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + + assert_error "WRONGTYPE*" {r SDIFFSTORE set3{t} set1{t} key1{t} set2{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + } + + test "SDIFFSTORE should handle non existing key as empty" { + r del set1{t} set2{t} set3{t} + + r set setres{t} xxx + assert_equal 0 [r sdiffstore setres{t} foo111{t} bar222{t}] + assert_equal 0 [r exists setres{t}] + + # with a legal dstkey, should delete dstkey + r sadd set3{t} a b c + assert_equal 0 [r sdiffstore set3{t} set1{t} set2{t}] + assert_equal 0 [r exists set3{t}] + + r sadd set1{t} a b c + assert_equal 3 [r sdiffstore set3{t} set1{t} set2{t}] + assert_equal 1 [r exists set3{t}] + assert_equal {a b c} [lsort [r smembers set3{t}]] + + # with a legal dstkey and empty set2, should delete the dstkey + r sadd set3{t} a b c + assert_equal 0 [r sdiffstore set3{t} set2{t} set1{t}] + assert_equal 0 [r exists set3{t}] + } + + test "SINTER against non-set should throw error" { + r set key1{t} x + assert_error "WRONGTYPE*" {r sinter key1{t} noset{t}} + # different order + assert_error "WRONGTYPE*" {r sinter noset{t} key1{t}} + + r sadd set1{t} a b c + assert_error "WRONGTYPE*" {r sinter key1{t} set1{t}} + # different order + assert_error "WRONGTYPE*" {r sinter set1{t} key1{t}} } test "SINTER should handle non existing key as empty" { @@ -303,10 +375,115 @@ start_server { lsort [r sinter set1 set2] } {1 2 3} + test "SINTERSTORE against non-set should throw error" { + r del set1{t} set2{t} set3{t} key1{t} + r set key1{t} x + + # with en empty dstkey + assert_error "WRONGTYPE*" {r sinterstore set3{t} key1{t} noset{t}} + assert_equal 0 [r exists set3{t}] + assert_error "WRONGTYPE*" {r sinterstore set3{t} noset{t} key1{t}} + assert_equal 0 [r exists set3{t}] + + # with a legal dstkey + r sadd set1{t} a b c + r sadd set2{t} b c d + r sadd set3{t} e + assert_error "WRONGTYPE*" {r sinterstore set3{t} key1{t} set2{t} noset{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + + assert_error "WRONGTYPE*" {r sinterstore set3{t} noset{t} key1{t} set2{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + } + test "SINTERSTORE against non existing keys should delete dstkey" { - r set setres xxx - assert_equal 0 [r sinterstore setres foo111 bar222] - assert_equal 0 [r exists setres] + r del set1{t} set2{t} set3{t} + + r set setres{t} xxx + assert_equal 0 [r sinterstore setres{t} foo111{t} bar222{t}] + assert_equal 0 [r exists setres{t}] + + # with a legal dstkey + r sadd set3{t} a b c + assert_equal 0 [r sinterstore set3{t} set1{t} set2{t}] + assert_equal 0 [r exists set3{t}] + + r sadd set1{t} a b c + assert_equal 0 [r sinterstore set3{t} set1{t} set2{t}] + assert_equal 0 [r exists set3{t}] + + assert_equal 0 [r sinterstore set3{t} set2{t} set1{t}] + assert_equal 0 [r exists set3{t}] + } + + test "SUNION against non-set should throw error" { + r set key1{t} x + assert_error "WRONGTYPE*" {r sunion key1{t} noset{t}} + # different order + assert_error "WRONGTYPE*" {r sunion noset{t} key1{t}} + + r del set1{t} + r sadd set1{t} a b c + assert_error "WRONGTYPE*" {r sunion key1{t} set1{t}} + # different order + assert_error "WRONGTYPE*" {r sunion set1{t} key1{t}} + } + + test "SUNION should handle non existing key as empty" { + r del set1{t} set2{t} set3{t} + + r sadd set1{t} a b c + r sadd set2{t} b c d + assert_equal {a b c d} [lsort [r sunion set1{t} set2{t} set3{t}]] + } + + test "SUNIONSTORE against non-set should throw error" { + r del set1{t} set2{t} set3{t} key1{t} + r set key1{t} x + + # with en empty dstkey + assert_error "WRONGTYPE*" {r sunionstore set3{t} key1{t} noset{t}} + assert_equal 0 [r exists set3{t}] + assert_error "WRONGTYPE*" {r sunionstore set3{t} noset{t} key1{t}} + assert_equal 0 [r exists set3{t}] + + # with a legal dstkey + r sadd set1{t} a b c + r sadd set2{t} b c d + r sadd set3{t} e + assert_error "WRONGTYPE*" {r sunionstore set3{t} key1{t} key2{t} noset{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + + assert_error "WRONGTYPE*" {r sunionstore set3{t} noset{t} key1{t} key2{t}} + assert_equal 1 [r exists set3{t}] + assert_equal {e} [lsort [r smembers set3{t}]] + } + + test "SUNIONSTORE should handle non existing key as empty" { + r del set1{t} set2{t} set3{t} + + r set setres{t} xxx + assert_equal 0 [r sunionstore setres{t} foo111{t} bar222{t}] + assert_equal 0 [r exists setres{t}] + + # set1 set2 both empty, should delete the dstkey + r sadd set3{t} a b c + assert_equal 0 [r sunionstore set3{t} set1{t} set2{t}] + assert_equal 0 [r exists set3{t}] + + r sadd set1{t} a b c + r sadd set3{t} e f + assert_equal 3 [r sunionstore set3{t} set1{t} set2{t}] + assert_equal 1 [r exists set3{t}] + assert_equal {a b c} [lsort [r smembers set3{t}]] + + r sadd set3{t} d + assert_equal 3 [r sunionstore set3{t} set2{t} set1{t}] + assert_equal 1 [r exists set3{t}] + assert_equal {a b c} [lsort [r smembers set3{t}]] } test "SUNIONSTORE against non existing keys should delete dstkey" { From 4079f799745bd9e030b2c75d29373e6a13a0f015 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Wed, 2 Jun 2021 22:30:08 +0800 Subject: [PATCH 35/79] redis-cli cluster import command may issue wrong MIGRATE command. (#8945) In clusterManagerCommandImport strcat was used to concat COPY and REPLACE, the space maybe not enough. If we use --cluster-replace but not --cluster-copy, the MIGRATE command contained COPY instead of REPLACE. (cherry picked from commit a049f6295a28a20b11eff89083e91dab0738413b) --- src/redis-cli.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 06ccf1563..3160ada46 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -6540,9 +6540,9 @@ static int clusterManagerCommandImport(int argc, char **argv) { } if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_COPY) - strcat(cmdfmt, " %s"); + cmdfmt = sdscat(cmdfmt," COPY"); if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_REPLACE) - strcat(cmdfmt, " %s"); + cmdfmt = sdscat(cmdfmt," REPLACE"); /* Use SCAN to iterate over the keys, migrating to the * right node as needed. */ @@ -6574,8 +6574,7 @@ static int clusterManagerCommandImport(int argc, char **argv) { printf("Migrating %s to %s:%d: ", key, target->ip, target->port); redisReply *r = reconnectingRedisCommand(src_ctx, cmdfmt, target->ip, target->port, - key, 0, timeout, - "COPY", "REPLACE"); + key, 0, timeout); if (!r || r->type == REDIS_REPLY_ERROR) { if (r && r->str) { clusterManagerLogErr("Source %s:%d replied with " From ffe1e9107c3c8ab2bdcd0564d96e4307343ebac2 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Thu, 1 Jul 2021 17:11:27 +0300 Subject: [PATCH 36/79] Fix CLIENT UNBLOCK crashing modules. (#9167) Modules that use background threads with thread safe contexts are likely to use RM_BlockClient() without a timeout function, because they do not set up a timeout. Before this commit, `CLIENT UNBLOCK` would result with a crash as the `NULL` timeout callback is called. Beyond just crashing, this is also logically wrong as it may throw the module into an unexpected client state. This commits makes `CLIENT UNBLOCK` on such clients behave the same as any other client that is not in a blocked state and therefore cannot be unblocked. (cherry picked from commit aa139e2f02292d668370afde8c91575363c2d611) --- src/module.c | 21 ++++++- src/networking.c | 2 +- src/server.h | 1 + tests/modules/blockonbackground.c | 69 ++++++++++++++++++++++ tests/unit/moduleapi/blockonbackground.tcl | 29 +++++++++ 5 files changed, 119 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 720befe2c..c5de4b787 100644 --- a/src/module.c +++ b/src/module.c @@ -5367,8 +5367,8 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { * reply_callback: called after a successful RedisModule_UnblockClient() * call in order to reply to the client and unblock it. * - * timeout_callback: called when the timeout is reached in order to send an - * error to the client. + * timeout_callback: called when the timeout is reached or if `CLIENT UNBLOCK` + * is invoked, in order to send an error to the client. * * free_privdata: called in order to free the private data that is passed * by RedisModule_UnblockClient() call. @@ -5385,6 +5385,12 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) { * In these cases, a call to RedisModule_BlockClient() will **not** block the * client, but instead produce a specific error reply. * + * A module that registers a timeout_callback function can also be unblocked + * using the `CLIENT UNBLOCK` command, which will trigger the timeout callback. + * If a callback function is not registered, then the blocked client will be + * treated as if it is not in a blocked state and `CLIENT UNBLOCK` will return + * a zero value. + * * Measuring background time: By default the time spent in the blocked command * is not account for the total command duration. To include such time you should * use RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd() one, @@ -5654,6 +5660,17 @@ void moduleHandleBlockedClients(void) { pthread_mutex_unlock(&moduleUnblockedClientsMutex); } +/* Check if the specified client can be safely timed out using + * moduleBlockedClientTimedOut(). + */ +int moduleBlockedClientMayTimeout(client *c) { + if (c->btype != BLOCKED_MODULE) + return 1; + + RedisModuleBlockedClient *bc = c->bpop.module_blocked_handle; + return (bc && bc->timeout_callback != NULL); +} + /* Called when our client timed out. After this function unblockClient() * is called, and it will invalidate the blocked client. So this function * does not need to do any cleanup. Eventually the module will call the diff --git a/src/networking.c b/src/networking.c index 51c58ff20..ed092b0e0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -2677,7 +2677,7 @@ NULL if (getLongLongFromObjectOrReply(c,c->argv[2],&id,NULL) != C_OK) return; struct client *target = lookupClientByID(id); - if (target && target->flags & CLIENT_BLOCKED) { + if (target && target->flags & CLIENT_BLOCKED && moduleBlockedClientMayTimeout(target)) { if (unblock_error) addReplyError(target, "-UNBLOCKED client unblocked via CLIENT UNBLOCK"); diff --git a/src/server.h b/src/server.h index f57a720e9..67541fe60 100644 --- a/src/server.h +++ b/src/server.h @@ -1779,6 +1779,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data); void processModuleLoadingProgressEvent(int is_aof); int moduleTryServeClientBlockedOnKey(client *c, robj *key); void moduleUnblockClient(client *c); +int moduleBlockedClientMayTimeout(client *c); int moduleClientIsBlockedOnKeys(client *c); void moduleNotifyUserChanged(client *c); void moduleNotifyKeyUnlink(robj *key, robj *val); diff --git a/tests/modules/blockonbackground.c b/tests/modules/blockonbackground.c index 855fef9dc..bc6845a85 100644 --- a/tests/modules/blockonbackground.c +++ b/tests/modules/blockonbackground.c @@ -195,6 +195,63 @@ int HelloDoubleBlock_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, return REDISMODULE_OK; } +RedisModuleBlockedClient *blocked_client = NULL; + +/* BLOCK.BLOCK [TIMEOUT] -- Blocks the current client until released + * or TIMEOUT seconds. If TIMEOUT is zero, no timeout function is + * registered. + */ +int Block_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (RedisModule_IsBlockedReplyRequest(ctx)) { + RedisModuleString *r = RedisModule_GetBlockedClientPrivateData(ctx); + return RedisModule_ReplyWithString(ctx, r); + } else if (RedisModule_IsBlockedTimeoutRequest(ctx)) { + blocked_client = NULL; + return RedisModule_ReplyWithSimpleString(ctx, "Timed out"); + } + + if (argc != 2) return RedisModule_WrongArity(ctx); + long long timeout; + + if (RedisModule_StringToLongLong(argv[1], &timeout) != REDISMODULE_OK) { + return RedisModule_ReplyWithError(ctx, "ERR invalid timeout"); + } + if (blocked_client) { + return RedisModule_ReplyWithError(ctx, "ERR another client already blocked"); + } + + /* Block client. We use this function as both a reply and optional timeout + * callback and differentiate the different code flows above. + */ + blocked_client = RedisModule_BlockClient(ctx, Block_RedisCommand, + timeout > 0 ? Block_RedisCommand : NULL, NULL, timeout); + return REDISMODULE_OK; +} + +/* BLOCK.IS_BLOCKED -- Returns 1 if we have a blocked client, or 0 otherwise. + */ +int IsBlocked_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + UNUSED(argv); + UNUSED(argc); + RedisModule_ReplyWithLongLong(ctx, blocked_client ? 1 : 0); + return REDISMODULE_OK; +} + +/* BLOCK.RELEASE [reply] -- Releases the blocked client and produce the specified reply. + */ +int Release_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + if (argc != 2) return RedisModule_WrongArity(ctx); + if (!blocked_client) { + return RedisModule_ReplyWithError(ctx, "ERR No blocked client"); + } + + RedisModule_UnblockClient(blocked_client, argv[1]); + blocked_client = NULL; + + RedisModule_ReplyWithSimpleString(ctx, "OK"); + + return REDISMODULE_OK; +} int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { UNUSED(argv); @@ -215,5 +272,17 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) HelloBlockNoTracking_RedisCommand,"",0,0,0) == REDISMODULE_ERR) return REDISMODULE_ERR; + if (RedisModule_CreateCommand(ctx, "block.block", + Block_RedisCommand, "", 0, 0, 0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"block.is_blocked", + IsBlocked_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + + if (RedisModule_CreateCommand(ctx,"block.release", + Release_RedisCommand,"",0,0,0) == REDISMODULE_ERR) + return REDISMODULE_ERR; + return REDISMODULE_OK; } diff --git a/tests/unit/moduleapi/blockonbackground.tcl b/tests/unit/moduleapi/blockonbackground.tcl index 66a232fab..79ca52143 100644 --- a/tests/unit/moduleapi/blockonbackground.tcl +++ b/tests/unit/moduleapi/blockonbackground.tcl @@ -85,4 +85,33 @@ start_server {tags {"modules"}} { assert_equal [r slowlog len] 0 } } + + test "client unblock works only for modules with timeout support" { + set rd [redis_deferring_client] + $rd client id + set id [$rd read] + + # Block with a timeout function - may unblock + $rd block.block 20000 + wait_for_condition 50 100 { + [r block.is_blocked] == 1 + } else { + fail "Module did not block" + } + + assert_equal 1 [r client unblock $id] + assert_match {*Timed out*} [$rd read] + + # Block without a timeout function - cannot unblock + $rd block.block 0 + wait_for_condition 50 100 { + [r block.is_blocked] == 1 + } else { + fail "Module did not block" + } + + assert_equal 0 [r client unblock $id] + assert_equal "OK" [r block.release foobar] + assert_equal "foobar" [$rd read] + } } From 501bcb9ad54aef58aa03466583960344443e5ad6 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 4 Jul 2021 14:21:53 +0300 Subject: [PATCH 37/79] fix valgrind issues with recently added test in modules/blockonbackground (#9192) fixes test issue introduced in #9167 1. invalid reads due to accessing non-retained string (passed as unblock context). 2. leaking module blocked client context, see #6922 for info. (cherry picked from commit a8518cce951629eaccde40fd0e51b36a5dc6321c) --- tests/modules/blockonbackground.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/modules/blockonbackground.c b/tests/modules/blockonbackground.c index bc6845a85..688756309 100644 --- a/tests/modules/blockonbackground.c +++ b/tests/modules/blockonbackground.c @@ -206,6 +206,7 @@ int Block_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) RedisModuleString *r = RedisModule_GetBlockedClientPrivateData(ctx); return RedisModule_ReplyWithString(ctx, r); } else if (RedisModule_IsBlockedTimeoutRequest(ctx)) { + RedisModule_UnblockClient(blocked_client, NULL); /* Must be called to avoid leaks. */ blocked_client = NULL; return RedisModule_ReplyWithSimpleString(ctx, "Timed out"); } @@ -245,7 +246,9 @@ int Release_RedisCommand(RedisModuleCtx *ctx, RedisModuleString **argv, int argc return RedisModule_ReplyWithError(ctx, "ERR No blocked client"); } - RedisModule_UnblockClient(blocked_client, argv[1]); + RedisModuleString *replystr = argv[1]; + RedisModule_RetainString(ctx, replystr); + int err = RedisModule_UnblockClient(blocked_client, replystr); blocked_client = NULL; RedisModule_ReplyWithSimpleString(ctx, "OK"); From fa902f18be7f98845c2cd8c426238b568d035775 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Wed, 14 Jul 2021 12:32:13 +0300 Subject: [PATCH 38/79] Fix compatibility with OpenSSL 1.1.0. (#9233) (cherry picked from commit 277e4dc2032356c7712b539e89f7e9154e0a1a86) --- src/tls.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tls.c b/src/tls.c index ffd3b0ad0..0f76256d2 100644 --- a/src/tls.c +++ b/src/tls.c @@ -146,6 +146,8 @@ void tlsInit(void) { */ #if OPENSSL_VERSION_NUMBER < 0x10100000L OPENSSL_config(NULL); + #elif OPENSSL_VERSION_NUMBER < 0x10101000L + OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL); #endif From 4edf50d8e2ff93ca3ebfd5dce8af06c61fecd366 Mon Sep 17 00:00:00 2001 From: qetu3790 Date: Fri, 16 Jul 2021 11:40:25 +0800 Subject: [PATCH 39/79] Set TCP keepalive on inbound clusterbus connections (#9230) Set TCP keepalive on inbound clusterbus connections to prevent memory leak (cherry picked from commit f03af47a34ec672a7d9b18150a5be3a83681c19b) --- src/cluster.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cluster.c b/src/cluster.c index 28650b327..246eb4050 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -707,6 +707,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } connNonBlock(conn); connEnableTcpNoDelay(conn); + connKeepAlive(conn,server.cluster_node_timeout * 2); /* Use non-blocking I/O for cluster messages. */ serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport); From 72b8db27c8c84c4c553ad1fd8245d1f0c671730d Mon Sep 17 00:00:00 2001 From: Binbin Date: Sat, 17 Jul 2021 14:54:06 +0800 Subject: [PATCH 40/79] SMOVE only notify dstset when the addition is successful. (#9244) in case dest key already contains the member, the dest key isn't modified, so the command shouldn't invalidate watch. (cherry picked from commit 11dc4e59b365d6cd8699604d7d1c1025b6bb6259) --- src/t_set.c | 2 +- tests/unit/type/set.tcl | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index d0c54848e..2bce6294d 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -392,12 +392,12 @@ void smoveCommand(client *c) { } signalModifiedKey(c,c->db,c->argv[1]); - signalModifiedKey(c,c->db,c->argv[2]); server.dirty++; /* An extra key has changed when ele was successfully added to dstset */ if (setTypeAdd(dstset,ele->ptr)) { server.dirty++; + signalModifiedKey(c,c->db,c->argv[2]); notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id); } addReply(c,shared.cone); diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index bc5accb41..ee7b936b5 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -826,6 +826,28 @@ start_server { lsort [r smembers set] } {a b c} + test "SMOVE only notify dstset when the addition is successful" { + r del srcset{t} + r del dstset{t} + + r sadd srcset{t} a b + r sadd dstset{t} a + + r watch dstset{t} + + r multi + r sadd dstset{t} c + + set r2 [redis_client] + $r2 smove srcset{t} dstset{t} a + + # The dstset is actually unchanged, multi should success + r exec + set res [r scard dstset{t}] + assert_equal $res 2 + $r2 close + } + tags {slow} { test {intsets implementation stress testing} { for {set j 0} {$j < 20} {incr j} { From 05376f578abe5f423533609b0ef5b5bd390fbd07 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Mon, 19 Jul 2021 16:10:25 +0800 Subject: [PATCH 41/79] Fix missing separator in module info line (usedby and using lists) (#9241) Fix module info genModulesInfoStringRenderModulesList lack separator when there's more than one module in the list. Co-authored-by: Oran Agra (cherry picked from commit 1895e134a77efd789b1a6daee76a6ba5ec90e516) --- src/module.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/module.c b/src/module.c index c5de4b787..94107362b 100644 --- a/src/module.c +++ b/src/module.c @@ -8701,8 +8701,9 @@ sds genModulesInfoStringRenderModulesList(list *l) { while((ln = listNext(&li))) { RedisModule *module = ln->value; output = sdscat(output,module->name); + if (ln != listLast(l)) + output = sdscat(output,"|"); } - output = sdstrim(output,"|"); output = sdscat(output,"]"); return output; } From 15a7795b5e0372ea29de4fd2927d0cda45663c16 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 22 Jun 2021 17:22:40 +0300 Subject: [PATCH 42/79] Adjustments to recent RM_StringTruncate fix (#3718) (#9125) - Introduce a new sdssubstr api as a building block for sdsrange. The API of sdsrange is many times hard to work with and also has corner case that cause bugs. sdsrange is easy to work with and also simplifies the implementation of sdsrange. - Revert the fix to RM_StringTruncate and just use sdssubstr instead of sdsrange. - Solve valgrind warnings from the new tests introduced by the previous PR. (cherry picked from commit ae418eca24ba53a7dca07b0e7065f856e625469b) --- src/module.c | 21 ++++++---------- src/sds.c | 57 +++++++++++++++++++++++++++--------------- src/sds.h | 1 + tests/modules/basics.c | 3 +++ 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/src/module.c b/src/module.c index 94107362b..bf6580a60 100644 --- a/src/module.c +++ b/src/module.c @@ -2538,19 +2538,14 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { } else { /* Unshare and resize. */ key->value = dbUnshareStringValue(key->db, key->key, key->value); - if (newlen == 0) { - sdsfree(key->value->ptr); - key->value->ptr = sdsempty(); - } else { - size_t curlen = sdslen(key->value->ptr); - if (newlen > curlen) { - key->value->ptr = sdsgrowzero(key->value->ptr,newlen); - } else if (newlen < curlen) { - sdsrange(key->value->ptr,0,newlen-1); - /* If the string is too wasteful, reallocate it. */ - if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) - key->value->ptr = sdsRemoveFreeSpace(key->value->ptr); - } + size_t curlen = sdslen(key->value->ptr); + if (newlen > curlen) { + key->value->ptr = sdsgrowzero(key->value->ptr,newlen); + } else if (newlen < curlen) { + sdssubstr(key->value->ptr,0,newlen); + /* If the string is too wasteful, reallocate it. */ + if (sdslen(key->value->ptr) < sdsavail(key->value->ptr)) + key->value->ptr = sdsRemoveFreeSpace(key->value->ptr); } } return REDISMODULE_OK; diff --git a/src/sds.c b/src/sds.c index 2ec3aa733..8d1c2dbd0 100644 --- a/src/sds.c +++ b/src/sds.c @@ -759,6 +759,21 @@ sds sdstrim(sds s, const char *cset) { return s; } +/* Changes the input string to be a subset of the original. + * It does not release the free space in the string, so a call to + * sdsRemoveFreeSpace may be wise after. */ +void sdssubstr(sds s, size_t start, size_t len) { + /* Clamp out of range input */ + size_t oldlen = sdslen(s); + if (start >= oldlen) start = len = 0; + if (len > oldlen-start) len = oldlen-start; + + /* Move the data */ + if (len) memmove(s, s+start, len); + s[len] = 0; + sdssetlen(s,len); +} + /* Turn the string into a smaller (or equal) string containing only the * substring specified by the 'start' and 'end' indexes. * @@ -770,6 +785,11 @@ sds sdstrim(sds s, const char *cset) { * * The string is modified in-place. * + * NOTE: this function can be misleading and can have unexpected behaviour, + * specifically when you want the length of the new string to be 0. + * Having start==end will result in a string with one character. + * please consider using sdssubstr instead. + * * Example: * * s = sdsnew("Hello World"); @@ -777,28 +797,13 @@ sds sdstrim(sds s, const char *cset) { */ void sdsrange(sds s, ssize_t start, ssize_t end) { size_t newlen, len = sdslen(s); - if (len == 0) return; - if (start < 0) { - start = len+start; - if (start < 0) start = 0; - } - if (end < 0) { - end = len+end; - if (end < 0) end = 0; - } + if (start < 0) + start = len + start; + if (end < 0) + end = len + end; newlen = (start > end) ? 0 : (end-start)+1; - if (newlen != 0) { - if (start >= (ssize_t)len) { - newlen = 0; - } else if (end >= (ssize_t)len) { - end = len-1; - newlen = (start > end) ? 0 : (end-start)+1; - } - } - if (start && newlen) memmove(s, s+start, newlen); - s[newlen] = 0; - sdssetlen(s,newlen); + sdssubstr(s, start, newlen); } /* Apply tolower() to every character of the sds string 's'. */ @@ -1353,6 +1358,18 @@ int sdsTest(int argc, char **argv, int accurate) { test_cond("sdsrange(...,100,100)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0); + sdsfree(y); + y = sdsdup(x); + sdsrange(y,4,6); + test_cond("sdsrange(...,4,6)", + sdslen(y) == 0 && memcmp(y,"\0",1) == 0); + + sdsfree(y); + y = sdsdup(x); + sdsrange(y,3,6); + test_cond("sdsrange(...,3,6)", + sdslen(y) == 1 && memcmp(y,"o\0",2) == 0); + sdsfree(y); sdsfree(x); x = sdsnew("foo"); diff --git a/src/sds.h b/src/sds.h index 7f8710745..6699858ce 100644 --- a/src/sds.h +++ b/src/sds.h @@ -238,6 +238,7 @@ sds sdscatprintf(sds s, const char *fmt, ...); sds sdscatfmt(sds s, char const *fmt, ...); sds sdstrim(sds s, const char *cset); +void sdssubstr(sds s, size_t start, size_t len); void sdsrange(sds s, ssize_t start, ssize_t end); void sdsupdatelen(sds s); void sdsclear(sds s); diff --git a/tests/modules/basics.c b/tests/modules/basics.c index 9786be1b9..2618f45b8 100644 --- a/tests/modules/basics.c +++ b/tests/modules/basics.c @@ -156,6 +156,7 @@ int TestUnlink(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { /* TEST.STRING.TRUNCATE -- Test truncating an existing string object. */ int TestStringTruncate(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + RedisModule_AutoMemory(ctx); REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); @@ -208,6 +209,7 @@ int TestStringTruncate(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, RedisModuleString *key) { + RedisModule_AutoMemory(ctx); /* Increment a counter on the notifications: for each key notified we * increment a counter */ RedisModule_Log(ctx, "notice", "Got event type %d, event %s, key %s", type, @@ -219,6 +221,7 @@ int NotifyCallback(RedisModuleCtx *ctx, int type, const char *event, /* TEST.NOTIFICATIONS -- Test Keyspace Notifications. */ int TestNotifications(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { + RedisModule_AutoMemory(ctx); REDISMODULE_NOT_USED(argv); REDISMODULE_NOT_USED(argc); From 10cd7ff76a24b0b5a690d43de12b00b4f13ca9e3 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 24 Jun 2021 12:44:13 +0300 Subject: [PATCH 43/79] Fix failing basics moduleapi test on 32bit CI (#9140) (cherry picked from commit 5ffdbae1f64bb66b6e2470779540fb1051dcbff1) --- tests/modules/basics.c | 7 +++++++ tests/unit/moduleapi/basics.tcl | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/modules/basics.c b/tests/modules/basics.c index 2618f45b8..59ceb2d1d 100644 --- a/tests/modules/basics.c +++ b/tests/modules/basics.c @@ -333,6 +333,9 @@ int TestCtxFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { flags = RedisModule_GetContextFlags(ctx); if (!(flags & REDISMODULE_CTX_FLAGS_AOF)) FAIL("AOF Flag not set after config set"); + /* Disable RDB saving and test the flag. */ + RedisModule_Call(ctx, "config", "ccc", "set", "save", ""); + flags = RedisModule_GetContextFlags(ctx); if (flags & REDISMODULE_CTX_FLAGS_RDB) FAIL("RDB Flag was set"); /* Enable RDB to test RDB flags */ RedisModule_Call(ctx, "config", "ccc", "set", "save", "900 1"); @@ -344,8 +347,12 @@ int TestCtxFlags(RedisModuleCtx *ctx, RedisModuleString **argv, int argc) { if (flags & REDISMODULE_CTX_FLAGS_READONLY) FAIL("Read-only flag was set"); if (flags & REDISMODULE_CTX_FLAGS_CLUSTER) FAIL("Cluster flag was set"); + /* Disable maxmemory and test the flag. (it is implicitly set in 32bit builds. */ + RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory", "0"); + flags = RedisModule_GetContextFlags(ctx); if (flags & REDISMODULE_CTX_FLAGS_MAXMEMORY) FAIL("Maxmemory flag was set"); + /* Enable maxmemory and test the flag. */ RedisModule_Call(ctx, "config", "ccc", "set", "maxmemory", "100000000"); flags = RedisModule_GetContextFlags(ctx); if (!(flags & REDISMODULE_CTX_FLAGS_MAXMEMORY)) diff --git a/tests/unit/moduleapi/basics.tcl b/tests/unit/moduleapi/basics.tcl index 513ace6b9..96683f4cc 100644 --- a/tests/unit/moduleapi/basics.tcl +++ b/tests/unit/moduleapi/basics.tcl @@ -1,8 +1,7 @@ set testmodule [file normalize tests/modules/basics.so] -# TEST.CTXFLAGS requires RDB to be disabled, so override save file -start_server {tags {"modules"} overrides {save ""}} { +start_server {tags {"modules"}} { r module load $testmodule test {test module api basics} { From 74f84db309e6228bbfd8b1b6b2a138db04993667 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Mon, 19 Jul 2021 13:23:25 +0800 Subject: [PATCH 44/79] Remove testmodule in src/modules/Makefile. (#9250) src/modules make failed. As in #3718 testmodule.c was removed. But the makefile was not updated (cherry picked from commit d54c9086c267d20bb6981f5a60f589e93b662d62) --- src/modules/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/modules/Makefile b/src/modules/Makefile index 5e012d6f1..3db19e79a 100644 --- a/src/modules/Makefile +++ b/src/modules/Makefile @@ -13,7 +13,7 @@ endif .SUFFIXES: .c .so .xo .o -all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so +all: helloworld.so hellotype.so helloblock.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so .c.xo: $(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@ @@ -58,10 +58,5 @@ helloacl.xo: ../redismodule.h helloacl.so: helloacl.xo $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc -testmodule.xo: ../redismodule.h - -testmodule.so: testmodule.xo - $(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc - clean: rm -rf *.xo *.so From 4233b6097853f0e2161c9b72e3a09b14457ab56f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 18 May 2021 17:13:59 +0300 Subject: [PATCH 45/79] longer timeout in replication test (#8963) the test normally passes. but we saw one failure in a valgrind run in github actions (cherry picked from commit 8458baf6a96fa6c6050bac24160f82d32a0b9ed4) --- tests/integration/replication.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 0e8dbcf1b..916dd324e 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -900,7 +900,7 @@ test {Kill rdb child process if its dumping RDB is not useful} { # Slave2 disconnect with master $slave2 slaveof no one # Should kill child - wait_for_condition 20 10 { + wait_for_condition 100 10 { [s 0 rdb_bgsave_in_progress] eq 0 } else { fail "can't kill rdb child" From ad8562e5a1b7e85821f9a0c14646982ea790a6c5 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Wed, 21 Jul 2021 21:25:19 +0800 Subject: [PATCH 46/79] On 32 bit platform, the bit position of GETBIT/SETBIT/BITFIELD/BITCOUNT,BITPOS may overflow (see CVE-2021-32761) (#9191) GETBIT, SETBIT may access wrong address because of wrap. BITCOUNT and BITPOS may return wrapped results. BITFIELD may access the wrong address but also allocate insufficient memory and segfault (see CVE-2021-32761). This commit uses `uint64_t` or `long long` instead of `size_t`. related https://github.com/redis/redis/pull/8096 At 32bit platform: > setbit bit 4294967295 1 (integer) 0 > config set proto-max-bulk-len 536870913 OK > append bit "\xFF" (integer) 536870913 > getbit bit 4294967296 (integer) 0 When the bit index is larger than 4294967295, size_t can't hold bit index. In the past, `proto-max-bulk-len` is limit to 536870912, so there is no problem. After this commit, bit position is stored in `uint64_t` or `long long`. So when `proto-max-bulk-len > 536870912`, 32bit platforms can still be correct. For 64bit platform, this problem still exists. The major reason is bit pos 8 times of byte pos. When proto-max-bulk-len is very larger, bit pos may overflow. But at 64bit platform, we don't have so long string. So this bug may never happen. Additionally this commit add a test cost `512MB` memory which is tag as `large-memory`. Make freebsd ci and valgrind ci ignore this test. (cherry picked from commit 71d452876ebf8456afaadd6b3c27988abadd1148) --- .github/workflows/daily.yml | 6 +++--- src/bitops.c | 32 ++++++++++++++++---------------- src/server.h | 2 +- tests/unit/bitops.tcl | 28 ++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 9e4630e29..432971a9d 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -151,7 +151,7 @@ jobs: run: | sudo apt-get update sudo apt-get install tcl8.6 valgrind -y - ./runtest --valgrind --verbose --clients 1 --dump-logs + ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs - name: module api test run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 - name: unittest @@ -171,7 +171,7 @@ jobs: run: | sudo apt-get update sudo apt-get install tcl8.6 valgrind -y - ./runtest --valgrind --verbose --clients 1 --dump-logs + ./runtest --valgrind --verbose --clients 1 --tags -large-memory --dump-logs - name: module api test run: ./runtest-moduleapi --valgrind --no-latency --verbose --clients 1 @@ -260,7 +260,7 @@ jobs: prepare: pkg install -y bash gmake lang/tcl86 run: > gmake && - ./runtest --accurate --verbose --no-latency --dump-logs && + ./runtest --accurate --verbose --no-latency --tags -large-memory --dump-logs && MAKE=gmake ./runtest-moduleapi --verbose && ./runtest-sentinel && ./runtest-cluster diff --git a/src/bitops.c b/src/bitops.c index afd79ad88..f1c563a41 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -37,8 +37,8 @@ /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */ -size_t redisPopcount(void *s, long count) { - size_t bits = 0; +long long redisPopcount(void *s, long count) { + long long bits = 0; unsigned char *p = s; uint32_t *p4; static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; @@ -98,11 +98,11 @@ size_t redisPopcount(void *s, long count) { * no zero bit is found, it returns count*8 assuming the string is zero * padded on the right. However if 'bit' is 1 it is possible that there is * not a single set bit in the bitmap. In this special case -1 is returned. */ -long redisBitpos(void *s, unsigned long count, int bit) { +long long redisBitpos(void *s, unsigned long count, int bit) { unsigned long *l; unsigned char *c; unsigned long skipval, word = 0, one; - long pos = 0; /* Position of bit, to return to the caller. */ + long long pos = 0; /* Position of bit, to return to the caller. */ unsigned long j; int found; @@ -410,7 +410,7 @@ void printBits(unsigned char *p, unsigned long count) { * If the 'hash' argument is true, and 'bits is positive, then the command * will also parse bit offsets prefixed by "#". In such a case the offset * is multiplied by 'bits'. This is useful for the BITFIELD command. */ -int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int bits) { +int getBitOffsetFromArgument(client *c, robj *o, uint64_t *offset, int hash, int bits) { long long loffset; char *err = "bit offset is not an integer or out of range"; char *p = o->ptr; @@ -435,7 +435,7 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b return C_ERR; } - *offset = (size_t)loffset; + *offset = loffset; return C_OK; } @@ -477,7 +477,7 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) { * so that the 'maxbit' bit can be addressed. The object is finally * returned. Otherwise if the key holds a wrong type NULL is returned and * an error is sent to the client. */ -robj *lookupStringForBitCommand(client *c, size_t maxbit) { +robj *lookupStringForBitCommand(client *c, uint64_t maxbit) { size_t byte = maxbit >> 3; robj *o = lookupKeyWrite(c->db,c->argv[1]); if (checkType(c,o,OBJ_STRING)) return NULL; @@ -527,7 +527,7 @@ unsigned char *getObjectReadOnlyString(robj *o, long *len, char *llbuf) { void setbitCommand(client *c) { robj *o; char *err = "bit is not an integer or out of range"; - size_t bitoffset; + uint64_t bitoffset; ssize_t byte, bit; int byteval, bitval; long on; @@ -566,7 +566,7 @@ void setbitCommand(client *c) { void getbitCommand(client *c) { robj *o; char llbuf[32]; - size_t bitoffset; + uint64_t bitoffset; size_t byte, bit; size_t bitval = 0; @@ -888,7 +888,7 @@ void bitposCommand(client *c) { addReplyLongLong(c, -1); } else { long bytes = end-start+1; - long pos = redisBitpos(p+start,bytes,bit); + long long pos = redisBitpos(p+start,bytes,bit); /* If we are looking for clear bits, and the user specified an exact * range with start-end, we can't consider the right of the range as @@ -897,11 +897,11 @@ void bitposCommand(client *c) { * So if redisBitpos() returns the first bit outside the range, * we return -1 to the caller, to mean, in the specified range there * is not a single "0" bit. */ - if (end_given && bit == 0 && pos == bytes*8) { + if (end_given && bit == 0 && pos == (long long)bytes<<3) { addReplyLongLong(c,-1); return; } - if (pos != -1) pos += start*8; /* Adjust for the bytes we skipped. */ + if (pos != -1) pos += (long long)start<<3; /* Adjust for the bytes we skipped. */ addReplyLongLong(c,pos); } } @@ -933,12 +933,12 @@ struct bitfieldOp { * GET subcommand is allowed, other subcommands will return an error. */ void bitfieldGeneric(client *c, int flags) { robj *o; - size_t bitoffset; + uint64_t bitoffset; int j, numops = 0, changes = 0; struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */ int owtype = BFOVERFLOW_WRAP; /* Overflow type. */ int readonly = 1; - size_t highest_write_offset = 0; + uint64_t highest_write_offset = 0; for (j = 2; j < c->argc; j++) { int remargs = c->argc-j-1; /* Remaining args other than current. */ @@ -1128,9 +1128,9 @@ void bitfieldGeneric(client *c, int flags) { * object boundaries. */ memset(buf,0,9); int i; - size_t byte = thisop->offset >> 3; + uint64_t byte = thisop->offset >> 3; for (i = 0; i < 9; i++) { - if (src == NULL || i+byte >= (size_t)strlen) break; + if (src == NULL || i+byte >= (uint64_t)strlen) break; buf[i] = src[i+byte]; } diff --git a/src/server.h b/src/server.h index 67541fe60..caf9df31c 100644 --- a/src/server.h +++ b/src/server.h @@ -1795,7 +1795,7 @@ void getRandomHexChars(char *p, size_t len); void getRandomBytes(unsigned char *p, size_t len); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); void exitFromChild(int retcode); -size_t redisPopcount(void *s, long count); +long long redisPopcount(void *s, long count); int redisSetProcTitle(char *title); int validateProcTitleTemplate(const char *template); int redisCommunicateSystemd(const char *sd_notify_msg); diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index 926f38295..534832974 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -349,3 +349,31 @@ start_server {tags {"bitops"}} { } } } + +start_server {tags {"bitops large-memory"}} { + test "BIT pos larger than UINT_MAX" { + set bytes [expr (1 << 29) + 1] + set bitpos [expr (1 << 32)] + set oldval [lindex [r config get proto-max-bulk-len] 1] + r config set proto-max-bulk-len $bytes + r setbit mykey $bitpos 1 + assert_equal $bytes [r strlen mykey] + assert_equal 1 [r getbit mykey $bitpos] + assert_equal [list 128 128 -1] [r bitfield mykey get u8 $bitpos set u8 $bitpos 255 get i8 $bitpos] + assert_equal $bitpos [r bitpos mykey 1] + assert_equal $bitpos [r bitpos mykey 1 [expr $bytes - 1]] + if {$::accurate} { + # set all bits to 1 + set mega [expr (1 << 23)] + set part [string repeat "\xFF" $mega] + for {set i 0} {$i < 64} {incr i} { + r setrange mykey [expr $i * $mega] $part + } + r setrange mykey [expr $bytes - 1] "\xFF" + assert_equal [expr $bitpos + 8] [r bitcount mykey] + assert_equal -1 [r bitpos mykey 0 0 [expr $bytes - 1]] + } + r config set proto-max-bulk-len $oldval + r del mykey + } {1} +} From 9294b505f9742c59e4c61397333d2dac61d24076 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 21 Jul 2021 16:37:05 +0300 Subject: [PATCH 47/79] Redis 6.2.5 --- 00-RELEASENOTES | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/version.h | 4 ++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 3fd620cf0..a5fb897ee 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,53 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.2.5 Released Wed Jul 21 16:32:19 IDT 2021 +================================================================================ + +Upgrade urgency: SECURITY, contains fixes to security issues that affect +authenticated client connections on 32-bit versions. MODERATE otherwise. + +Fix integer overflow in BITFIELD on 32-bit versions (CVE-2021-32761). +An integer overflow bug in Redis version 2.2 or newer can be exploited using the +BITFIELD command to corrupt the heap and potentially result with remote code +execution. + +Bug fixes that involve behavior changes: +* Change reply type for ZPOPMAX/MIN with count in RESP3 to nested array (#8981). + Was using a flat array like in RESP2 instead of a nested array like ZRANGE does. +* Fix reply type for HRANDFIELD and ZRANDMEMBER when key is missing (#9178). + Was using a null array instead of an empty array. +* Fix reply type for ZRANGESTORE when source key is missing (#9089). + Was using an empty array like ZRANGE instead of 0 (used in the STORE variant). + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162) +* Fix crash after CLIENT UNPAUSE when threaded I/O config is enabled (#9041) +* Fix XTRIM or XADD with LIMIT may delete more entries than the limit (#9048) +* Fix build issue with OpenSSL 1.1.0 (#9233) + +Other bug fixes: +* Fail EXEC command in case a watched key is expired (#9194) +* Fix SMOVE not to invalidate dest key (WATCH and tracking) when member already exists (#9244) +* Fix SINTERSTORE not to delete dest key when getting a wrong type error (#9032) +* Fix overflows on 32-bit versions in GETBIT, SETBIT, BITCOUNT, BITPOS, and BITFIELD (#9191) +* Improve MEMORY USAGE on stream keys (#9164) +* Set TCP keepalive on inbound cluster bus connections (#9230) +* Fix diskless replica loading to recover from RDB short read on module AUX data (#9199) +* Fix race in client side tracking (#9116) +* Fix ziplist length updates on big-endian platforms (#2080) + +CLI tools: +* redis-cli cluster import command may issue wrong MIGRATE command, sending COPY instead of REPLACE (#8945) +* redis-cli --rdb fixes when using "-" to write to stdout (#9136, #9135) +* redis-cli support for RESP3 set type in CSV and RAW output (#7338) + +Modules: +* Module API for getting current command name (#8792) +* Fix RM_StringTruncate when newlen is 0 (#3718) +* Fix CLIENT UNBLOCK crashing modules without timeout callback (#9167) + ================================================================================ Redis 6.2.4 Released Tue June 1 12:00:00 IST 2021 ================================================================================ diff --git a/src/version.h b/src/version.h index c355ecfed..cd2ff3a6e 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.2.4" -#define REDIS_VERSION_NUM 0x00060204 +#define REDIS_VERSION "6.2.5" +#define REDIS_VERSION_NUM 0x00060205 From 59c94dedb2b5c618e553af3fc35aa2d2fab57614 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 26 Sep 2021 15:42:17 +0300 Subject: [PATCH 48/79] Fix Integer overflow issue with intsets (CVE-2021-32687) The vulnerability involves changing the default set-max-intset-entries configuration parameter to a very large value and constructing specially crafted commands to manipulate sets --- src/intset.c | 3 ++- src/rdb.c | 4 +++- src/t_set.c | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/intset.c b/src/intset.c index 9ba13898d..e36685138 100644 --- a/src/intset.c +++ b/src/intset.c @@ -104,7 +104,8 @@ intset *intsetNew(void) { /* Resize the intset */ static intset *intsetResize(intset *is, uint32_t len) { - uint32_t size = len*intrev32ifbe(is->encoding); + uint64_t size = (uint64_t)len*intrev32ifbe(is->encoding); + assert(size <= SIZE_MAX - sizeof(intset)); is = zrealloc(is,sizeof(intset)+size); return is; } diff --git a/src/rdb.c b/src/rdb.c index e902388e6..53f67a72e 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1562,7 +1562,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; /* Use a regular set when there are too many entries. */ - if (len > server.set_max_intset_entries) { + size_t max_entries = server.set_max_intset_entries; + if (max_entries >= 1<<30) max_entries = 1<<30; + if (len > max_entries) { o = createSetObject(); /* It's faster to expand the dict to the right size asap in order * to avoid rehashing */ diff --git a/src/t_set.c b/src/t_set.c index 2bce6294d..a4fc5311a 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -66,7 +66,10 @@ int setTypeAdd(robj *subject, sds value) { if (success) { /* Convert to regular set when the intset contains * too many entries. */ - if (intsetLen(subject->ptr) > server.set_max_intset_entries) + size_t max_entries = server.set_max_intset_entries; + /* limit to 1G entries due to intset internals. */ + if (max_entries >= 1<<30) max_entries = 1<<30; + if (intsetLen(subject->ptr) > max_entries) setTypeConvert(subject,OBJ_ENCODING_HT); return 1; } From 7cd645a043e9a557940ecb8271f441c29614c5a3 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 26 Sep 2021 15:46:37 +0300 Subject: [PATCH 49/79] Fix redis-cli / redis-sential overflow on some platforms (CVE-2021-32762) The redis-cli command line tool and redis-sentinel service may be vulnerable to integer overflow when parsing specially crafted large multi-bulk network replies. This is a result of a vulnerability in the underlying hiredis library which does not perform an overflow check before calling the calloc() heap allocation function. This issue only impacts systems with heap allocators that do not perform their own overflow checks. Most modern systems do and are therefore not likely to be affected. Furthermore, by default redis-sentinel uses the jemalloc allocator which is also not vulnerable. --- deps/hiredis/hiredis.c | 1 + deps/hiredis/test.c | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 51f22a665..990f61960 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -174,6 +174,7 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) { return NULL; if (elements > 0) { + if (SIZE_MAX / sizeof(redisReply*) < elements) return NULL; /* Don't overflow */ r->element = hi_calloc(elements,sizeof(redisReply*)); if (r->element == NULL) { freeReplyObject(r); diff --git a/deps/hiredis/test.c b/deps/hiredis/test.c index 829536739..bdff74e88 100644 --- a/deps/hiredis/test.c +++ b/deps/hiredis/test.c @@ -498,6 +498,20 @@ static void test_reply_reader(void) { freeReplyObject(reply); redisReaderFree(reader); + test("Multi-bulk never overflows regardless of maxelements: "); + size_t bad_mbulk_len = (SIZE_MAX / sizeof(void *)) + 3; + char bad_mbulk_reply[100]; + snprintf(bad_mbulk_reply, sizeof(bad_mbulk_reply), "*%llu\r\n+asdf\r\n", + (unsigned long long) bad_mbulk_len); + + reader = redisReaderCreate(); + reader->maxelements = 0; /* Don't rely on default limit */ + redisReaderFeed(reader, bad_mbulk_reply, strlen(bad_mbulk_reply)); + ret = redisReaderGetReply(reader,&reply); + test_cond(ret == REDIS_ERR && strcasecmp(reader->errstr, "Out of memory") == 0); + freeReplyObject(reply); + redisReaderFree(reader); + #if LLONG_MAX > SIZE_MAX test("Set error when array > SIZE_MAX: "); reader = redisReaderCreate(); From c34f970be56d9710d1c5f049d99a39c74f4f82cc Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Wed, 9 Jun 2021 17:31:39 +0300 Subject: [PATCH 50/79] Prevent unauthenticated client from easily consuming lots of memory (CVE-2021-32675) This change sets a low limit for multibulk and bulk length in the protocol for unauthenticated connections, so that they can't easily cause redis to allocate massive amounts of memory by sending just a few characters on the network. The new limits are 10 arguments of 16kb each (instead of 1m of 512mb) --- src/networking.c | 17 +++++++++++++++++ src/server.c | 9 ++------- src/server.h | 1 + tests/unit/auth.tcl | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/networking.c b/src/networking.c index ed092b0e0..dcb0a9f24 100644 --- a/src/networking.c +++ b/src/networking.c @@ -107,6 +107,15 @@ static void clientSetDefaultAuth(client *c) { !(c->user->flags & USER_FLAG_DISABLED); } +int authRequired(client *c) { + /* Check if the user is authenticated. This check is skipped in case + * the default user is flagged as "nopass" and is active. */ + int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || + (DefaultUser->flags & USER_FLAG_DISABLED)) && + !c->authenticated; + return auth_required; +} + client *createClient(connection *conn) { client *c = zmalloc(sizeof(client)); @@ -1860,6 +1869,10 @@ int processMultibulkBuffer(client *c) { addReplyError(c,"Protocol error: invalid multibulk length"); setProtocolError("invalid mbulk count",c); return C_ERR; + } else if (ll > 10 && authRequired(c)) { + addReplyError(c, "Protocol error: unauthenticated multibulk length"); + setProtocolError("unauth mbulk count", c); + return C_ERR; } c->qb_pos = (newline-c->querybuf)+2; @@ -1907,6 +1920,10 @@ int processMultibulkBuffer(client *c) { addReplyError(c,"Protocol error: invalid bulk length"); setProtocolError("invalid bulk length",c); return C_ERR; + } else if (ll > 16384 && authRequired(c)) { + addReplyError(c, "Protocol error: unauthenticated bulk length"); + setProtocolError("unauth bulk length", c); + return C_ERR; } c->qb_pos = newline-c->querybuf+2; diff --git a/src/server.c b/src/server.c index 6c001b266..43cb81b85 100644 --- a/src/server.c +++ b/src/server.c @@ -4001,13 +4001,8 @@ int processCommand(client *c) { int is_may_replicate_command = (c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE)) || (c->cmd->proc == execCommand && (c->mstate.cmd_flags & (CMD_WRITE | CMD_MAY_REPLICATE))); - /* Check if the user is authenticated. This check is skipped in case - * the default user is flagged as "nopass" and is active. */ - int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) || - (DefaultUser->flags & USER_FLAG_DISABLED)) && - !c->authenticated; - if (auth_required) { - /* AUTH and HELLO and no auth modules are valid even in + if (authRequired(c)) { + /* AUTH and HELLO and no auth commands are valid even in * non-authenticated state. */ if (!(c->cmd->flags & CMD_NO_AUTH)) { rejectCommand(c,shared.noautherr); diff --git a/src/server.h b/src/server.h index caf9df31c..d9fef9552 100644 --- a/src/server.h +++ b/src/server.h @@ -1897,6 +1897,7 @@ void protectClient(client *c); void unprotectClient(client *c); void initThreadedIO(void); client *lookupClientByID(uint64_t id); +int authRequired(client *c); #ifdef __GNUC__ void addReplyErrorFormat(client *c, const char *fmt, ...) diff --git a/tests/unit/auth.tcl b/tests/unit/auth.tcl index b63cf0126..5997707c6 100644 --- a/tests/unit/auth.tcl +++ b/tests/unit/auth.tcl @@ -24,6 +24,22 @@ start_server {tags {"auth"} overrides {requirepass foobar}} { r set foo 100 r incr foo } {101} + + test {For unauthenticated clients multibulk and bulk length are limited} { + set rr [redis [srv "host"] [srv "port"] 0 $::tls] + $rr write "*100\r\n" + $rr flush + catch {[$rr read]} e + assert_match {*unauthenticated multibulk length*} $e + $rr close + + set rr [redis [srv "host"] [srv "port"] 0 $::tls] + $rr write "*1\r\n\$100000000\r\n" + $rr flush + catch {[$rr read]} e + assert_match {*unauthenticated bulk length*} $e + $rr close + } } start_server {tags {"auth_binary_password"}} { From 73436d82a652e989266567e912403298a14d0d8b Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Sun, 13 Jun 2021 14:29:20 +0300 Subject: [PATCH 51/79] Fix protocol parsing on 'ldbReplParseCommand' (CVE-2021-32672) The protocol parsing on 'ldbReplParseCommand' (LUA debugging) Assumed protocol correctness. This means that if the following is given: *1 $100 test The parser will try to read additional 94 unallocated bytes after the client buffer. This commit fixes this issue by validating that there are actually enough bytes to read. It also limits the amount of data that can be sent by the debugger client to 1M so the client will not be able to explode the memory. --- src/scripting.c | 29 +++++++++++++++++++++++++---- tests/unit/scripting.tcl | 15 +++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index b0602f7df..3e7bf068e 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -2081,7 +2081,8 @@ int ldbDelBreakpoint(int line) { /* Expect a valid multi-bulk command in the debugging client query buffer. * On success the command is parsed and returned as an array of SDS strings, * otherwise NULL is returned and there is to read more buffer. */ -sds *ldbReplParseCommand(int *argcp) { +sds *ldbReplParseCommand(int *argcp, char** err) { + static char* protocol_error = "protocol error"; sds *argv = NULL; int argc = 0; if (sdslen(ldb.cbuf) == 0) return NULL; @@ -2098,7 +2099,7 @@ sds *ldbReplParseCommand(int *argcp) { /* Seek and parse *\r\n. */ p = strchr(p,'*'); if (!p) goto protoerr; char *plen = p+1; /* Multi bulk len pointer. */ - p = strstr(p,"\r\n"); if (!p) goto protoerr; + p = strstr(p,"\r\n"); if (!p) goto keep_reading; *p = '\0'; p += 2; *argcp = atoi(plen); if (*argcp <= 0 || *argcp > 1024) goto protoerr; @@ -2107,12 +2108,16 @@ sds *ldbReplParseCommand(int *argcp) { argv = zmalloc(sizeof(sds)*(*argcp)); argc = 0; while(argc < *argcp) { + /* reached the end but there should be more data to read */ + if (*p == '\0') goto keep_reading; + if (*p != '$') goto protoerr; plen = p+1; /* Bulk string len pointer. */ - p = strstr(p,"\r\n"); if (!p) goto protoerr; + p = strstr(p,"\r\n"); if (!p) goto keep_reading; *p = '\0'; p += 2; int slen = atoi(plen); /* Length of this arg. */ if (slen <= 0 || slen > 1024) goto protoerr; + if ((size_t)(p + slen + 2 - copy) > sdslen(copy) ) goto keep_reading; argv[argc++] = sdsnewlen(p,slen); p += slen; /* Skip the already parsed argument. */ if (p[0] != '\r' || p[1] != '\n') goto protoerr; @@ -2122,6 +2127,8 @@ sds *ldbReplParseCommand(int *argcp) { return argv; protoerr: + *err = protocol_error; +keep_reading: sdsfreesplitres(argv,argc); sdsfree(copy); return NULL; @@ -2610,12 +2617,17 @@ void ldbMaxlen(sds *argv, int argc) { int ldbRepl(lua_State *lua) { sds *argv; int argc; + char* err = NULL; /* We continue processing commands until a command that should return * to the Lua interpreter is found. */ while(1) { - while((argv = ldbReplParseCommand(&argc)) == NULL) { + while((argv = ldbReplParseCommand(&argc, &err)) == NULL) { char buf[1024]; + if (err) { + lua_pushstring(lua, err); + lua_error(lua); + } int nread = connRead(ldb.conn,buf,sizeof(buf)); if (nread <= 0) { /* Make sure the script runs without user input since the @@ -2625,6 +2637,15 @@ int ldbRepl(lua_State *lua) { return C_ERR; } ldb.cbuf = sdscatlen(ldb.cbuf,buf,nread); + /* after 1M we will exit with an error + * so that the client will not blow the memory + */ + if (sdslen(ldb.cbuf) > 1<<20) { + sdsfree(ldb.cbuf); + ldb.cbuf = sdsempty(); + lua_pushstring(lua, "max client buffer reached"); + lua_error(lua); + } } /* Flush the old buffer. */ diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 0efe34cad..cd6646dce 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -923,3 +923,18 @@ start_server {tags {"scripting"}} { r eval {return 'hello'} 0 r eval {return 'hello'} 0 } + +start_server {tags {"scripting needs:debug external:skip"}} { + test {Test scripting debug protocol parsing} { + r script debug sync + r eval {return 'hello'} 0 + catch {r 'hello\0world'} e + assert_match {*Unknown Redis Lua debugger command*} $e + catch {r 'hello\0'} e + assert_match {*Unknown Redis Lua debugger command*} $e + catch {r '\0hello'} e + assert_match {*Unknown Redis Lua debugger command*} $e + catch {r '\0hello\0'} e + assert_match {*Unknown Redis Lua debugger command*} $e + } +} From b7834ea6e46a4be08dbe4044d03db86293e3d551 Mon Sep 17 00:00:00 2001 From: "meir@redislabs.com" Date: Sun, 13 Jun 2021 14:27:18 +0300 Subject: [PATCH 52/79] Fix invalid memory write on lua stack overflow {CVE-2021-32626} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When LUA call our C code, by default, the LUA stack has room for 20 elements. In most cases, this is more than enough but sometimes it's not and the caller must verify the LUA stack size before he pushes elements. On 3 places in the code, there was no verification of the LUA stack size. On specific inputs this missing verification could have lead to invalid memory write: 1. On 'luaReplyToRedisReply', one might return a nested reply that will explode the LUA stack. 2. On 'redisProtocolToLuaType', the Redis reply might be deep enough    to explode the LUA stack (notice that currently there is no such    command in Redis that returns such a nested reply, but modules might    do it) 3. On 'ldbRedis', one might give a command with enough arguments to    explode the LUA stack (all the arguments will be pushed to the LUA    stack) This commit is solving all those 3 issues by calling 'lua_checkstack' and verify that there is enough room in the LUA stack to push elements. In case 'lua_checkstack' returns an error (there is not enough room in the LUA stack and it's not possible to increase the stack), we will do the following: 1. On 'luaReplyToRedisReply', we will return an error to the user. 2. On 'redisProtocolToLuaType' we will exit with panic (we assume this scenario is rare because it can only happen with a module). 3. On 'ldbRedis', we return an error. --- src/scripting.c | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 3e7bf068e..360df500e 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -129,6 +129,16 @@ void sha1hex(char *digest, char *script, size_t len) { */ char *redisProtocolToLuaType(lua_State *lua, char* reply) { + + if (!lua_checkstack(lua, 5)) { + /* + * Increase the Lua stack if needed, to make sure there is enough room + * to push 5 elements to the stack. On failure, exit with panic. +         * Notice that we need, in the worst case, 5 elements because redisProtocolToLuaType_Aggregate +         * might push 5 elements to the Lua stack.*/ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } + char *p = reply; switch(*p) { @@ -221,6 +231,11 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) { if (atype == '%') { p = redisProtocolToLuaType(lua,p); } else { + if (!lua_checkstack(lua, 1)) { + /* Notice that here we need to check the stack again because the recursive + * call to redisProtocolToLuaType might have use the room allocated in the stack */ + serverPanic("lua stack limit reach when parsing redis.call reply"); + } lua_pushboolean(lua,1); } lua_settable(lua,-3); @@ -340,6 +355,17 @@ void luaSortArray(lua_State *lua) { /* Reply to client 'c' converting the top element in the Lua stack to a * Redis reply. As a side effect the element is consumed from the stack. */ void luaReplyToRedisReply(client *c, lua_State *lua) { + + if (!lua_checkstack(lua, 4)) { + /* Increase the Lua stack if needed to make sure there is enough room + * to push 4 elements to the stack. On failure, return error. +         * Notice that we need, in the worst case, 4 elements because returning a map might + * require push 4 elements to the Lua stack.*/ + addReplyErrorFormat(c, "reached lua stack limit"); + lua_pop(lua,1); /* pop the element from the stack */ + return; + } + int t = lua_type(lua,-1); switch(t) { @@ -363,6 +389,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { * field. */ /* Handle error reply. */ + /* we took care of the stack size on function start */ lua_pushstring(lua,"err"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -405,6 +432,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { if (t == LUA_TTABLE) { int maplen = 0; void *replylen = addReplyDeferredLen(c); + /* we took care of the stack size on function start */ lua_pushnil(lua); /* Use nil to start iteration. */ while (lua_next(lua,-2)) { /* Stack now: table, key, value */ @@ -427,6 +455,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { if (t == LUA_TTABLE) { int setlen = 0; void *replylen = addReplyDeferredLen(c); + /* we took care of the stack size on function start */ lua_pushnil(lua); /* Use nil to start iteration. */ while (lua_next(lua,-2)) { /* Stack now: table, key, true */ @@ -446,6 +475,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) { void *replylen = addReplyDeferredLen(c); int j = 1, mbulklen = 0; while(1) { + /* we took care of the stack size on function start */ lua_pushnumber(lua,j++); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -2561,6 +2591,17 @@ void ldbEval(lua_State *lua, sds *argv, int argc) { void ldbRedis(lua_State *lua, sds *argv, int argc) { int j, saved_rc = server.lua_replicate_commands; + if (!lua_checkstack(lua, argc + 1)) { + /* Increase the Lua stack if needed to make sure there is enough room + * to push 'argc + 1' elements to the stack. On failure, return error. +         * Notice that we need, in worst case, 'argc + 1' elements because we push all the arguments +         * given by the user (without the first argument) and we also push the 'redis' global table and +         * 'redis.call' function so: +         * (1 (redis table)) + (1 (redis.call function)) + (argc - 1 (all arguments without the first)) = argc + 1*/ + ldbLogRedisReply("max lua stack reached"); + return; + } + lua_getglobal(lua,"redis"); lua_pushstring(lua,"call"); lua_gettable(lua,-2); /* Stack: redis, redis.call */ From 24977cdff6931c9f859de1a8d1e0b2cff2a0be66 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 3 Jun 2021 12:10:02 +0300 Subject: [PATCH 53/79] Fix ziplist and listpack overflows and truncations (CVE-2021-32627, CVE-2021-32628) - fix possible heap corruption in ziplist and listpack resulting by trying to allocate more than the maximum size of 4GB. - prevent ziplist (hash and zset) from reaching size of above 1GB, will be converted to HT encoding, that's not a useful size. - prevent listpack (stream) from reaching size of above 1GB. - XADD will start a new listpack if the new record may cause the previous listpack to grow over 1GB. - XADD will respond with an error if a single stream record is over 1GB - List type (ziplist in quicklist) was truncating strings that were over 4GB, now it'll respond with an error. --- src/geo.c | 5 +- src/listpack.c | 2 +- src/module.c | 6 +- src/quicklist.c | 16 +++- src/rdb.c | 45 +++++++---- src/server.h | 2 +- src/t_hash.c | 13 +++- src/t_list.c | 29 +++++++ src/t_stream.c | 48 +++++++++--- src/t_zset.c | 62 +++++++++------ src/ziplist.c | 17 ++++- src/ziplist.h | 1 + tests/unit/violations.tcl | 156 ++++++++++++++++++++++++++++++++++++++ 13 files changed, 341 insertions(+), 61 deletions(-) create mode 100644 tests/unit/violations.tcl diff --git a/src/geo.c b/src/geo.c index 7c75738a2..893f78a7e 100644 --- a/src/geo.c +++ b/src/geo.c @@ -770,7 +770,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { robj *zobj; zset *zs; int i; - size_t maxelelen = 0; + size_t maxelelen = 0, totelelen = 0; if (returned_items) { zobj = createZsetObject(); @@ -785,13 +785,14 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { size_t elelen = sdslen(gp->member); if (maxelelen < elelen) maxelelen = elelen; + totelelen += elelen; znode = zslInsert(zs->zsl,score,gp->member); serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK); gp->member = NULL; } if (returned_items) { - zsetConvertToZiplistIfNeeded(zobj,maxelelen); + zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen); setKey(c,c->db,storekey,zobj); decrRefCount(zobj); notifyKeyspaceEvent(NOTIFY_ZSET,flags & GEOSEARCH ? "geosearchstore" : "georadiusstore",storekey, diff --git a/src/listpack.c b/src/listpack.c index ee256bad3..27622d4a5 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -313,7 +313,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui } else { if (size < 64) *enclen = 1+size; else if (size < 4096) *enclen = 2+size; - else *enclen = 5+size; + else *enclen = 5+(uint64_t)size; return LP_ENCODING_STRING; } } diff --git a/src/module.c b/src/module.c index bf6580a60..adca9dc9c 100644 --- a/src/module.c +++ b/src/module.c @@ -3319,6 +3319,7 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) { * - EDOM if the given ID was 0-0 or not greater than all other IDs in the * stream (only if the AUTOID flag is unset) * - EFBIG if the stream has reached the last possible ID + * - ERANGE if the elements are too large to be stored. */ int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, long numfields) { /* Validate args */ @@ -3362,8 +3363,9 @@ int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisM use_id_ptr = &use_id; } if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr) == C_ERR) { - /* ID not greater than all existing IDs in the stream */ - errno = EDOM; + /* Either the ID not greater than all existing IDs in the stream, or + * the elements are too large to be stored. either way, errno is already + * set by streamAppendItem. */ return REDISMODULE_ERR; } /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey() diff --git a/src/quicklist.c b/src/quicklist.c index 5a1e41dcc..a9f8b43b1 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -45,11 +45,16 @@ #define REDIS_STATIC static #endif -/* Optimization levels for size-based filling */ +/* Optimization levels for size-based filling. + * Note that the largest possible limit is 16k, so even if each record takes + * just one byte, it still won't overflow the 16 bit count field. */ static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536}; /* Maximum size in bytes of any multi-element ziplist. - * Larger values will live in their own isolated ziplists. */ + * Larger values will live in their own isolated ziplists. + * This is used only if we're limited by record count. when we're limited by + * size, the maximum limit is bigger, but still safe. + * 8k is a recommended / default size limit */ #define SIZE_SAFETY_LIMIT 8192 /* Minimum ziplist size in bytes for attempting compression. */ @@ -444,6 +449,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node, unsigned int new_sz = node->sz + sz + ziplist_overhead; if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill))) return 1; + /* when we return 1 above we know that the limit is a size limit (which is + * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */ else if (!sizeMeetsSafetyLimit(new_sz)) return 0; else if ((int)node->count < fill) @@ -463,6 +470,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, unsigned int merge_sz = a->sz + b->sz - 11; if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill))) return 1; + /* when we return 1 above we know that the limit is a size limit (which is + * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */ else if (!sizeMeetsSafetyLimit(merge_sz)) return 0; else if ((int)(a->count + b->count) <= fill) @@ -482,6 +491,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, * Returns 1 if new head created. */ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { quicklistNode *orig_head = quicklist->head; + assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ if (likely( _quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) { quicklist->head->zl = @@ -505,6 +515,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) { * Returns 1 if new tail created. */ int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) { quicklistNode *orig_tail = quicklist->tail; + assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ if (likely( _quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) { quicklist->tail->zl = @@ -847,6 +858,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry, int fill = quicklist->fill; quicklistNode *node = entry->node; quicklistNode *new_node = NULL; + assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */ if (!node) { /* we have no reference node, so let's create only node in the list */ diff --git a/src/rdb.c b/src/rdb.c index 53f67a72e..5456c1d80 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1625,7 +1625,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) { /* Read list/set value. */ uint64_t zsetlen; - size_t maxelelen = 0; + size_t maxelelen = 0, totelelen = 0; zset *zs; if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; @@ -1665,6 +1665,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { /* Don't care about integer-encoded strings. */ if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele); + totelelen += sdslen(sdsele); znode = zslInsert(zs->zsl,score,sdsele); if (dictAdd(zs->dict,sdsele,&znode->score) != DICT_OK) { @@ -1677,8 +1678,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { /* Convert *after* loading, since sorted sets are not stored ordered. */ if (zsetLength(o) <= server.zset_max_ziplist_entries && - maxelelen <= server.zset_max_ziplist_value) - zsetConvert(o,OBJ_ENCODING_ZIPLIST); + maxelelen <= server.zset_max_ziplist_value && + ziplistSafeToAdd(NULL, totelelen)) + { + zsetConvert(o,OBJ_ENCODING_ZIPLIST); + } } else if (rdbtype == RDB_TYPE_HASH) { uint64_t len; int ret; @@ -1731,21 +1735,30 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } } + /* Convert to hash table if size threshold is exceeded */ + if (sdslen(field) > server.hash_max_ziplist_value || + sdslen(value) > server.hash_max_ziplist_value || + !ziplistSafeToAdd(o->ptr, sdslen(field)+sdslen(value))) + { + hashTypeConvert(o, OBJ_ENCODING_HT); + ret = dictAdd((dict*)o->ptr, field, value); + if (ret == DICT_ERR) { + rdbReportCorruptRDB("Duplicate hash fields detected"); + if (dupSearchDict) dictRelease(dupSearchDict); + sdsfree(value); + sdsfree(field); + decrRefCount(o); + return NULL; + } + break; + } + /* Add pair to ziplist */ o->ptr = ziplistPush(o->ptr, (unsigned char*)field, sdslen(field), ZIPLIST_TAIL); o->ptr = ziplistPush(o->ptr, (unsigned char*)value, sdslen(value), ZIPLIST_TAIL); - /* Convert to hash table if size threshold is exceeded */ - if (sdslen(field) > server.hash_max_ziplist_value || - sdslen(value) > server.hash_max_ziplist_value) - { - sdsfree(field); - sdsfree(value); - hashTypeConvert(o, OBJ_ENCODING_HT); - break; - } sdsfree(field); sdsfree(value); } @@ -1858,12 +1871,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) { if (flen > maxlen) maxlen = flen; if (vlen > maxlen) maxlen = vlen; - zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); - zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); /* search for duplicate records */ sds field = sdstrynewlen(fstr, flen); - if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK) { + if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK || + !ziplistSafeToAdd(zl, (size_t)flen + vlen)) { rdbReportCorruptRDB("Hash zipmap with dup elements, or big length (%u)", flen); dictRelease(dupSearchDict); sdsfree(field); @@ -1872,6 +1884,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { decrRefCount(o); return NULL; } + + zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL); + zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL); } dictRelease(dupSearchDict); diff --git a/src/server.h b/src/server.h index d9fef9552..07b34c743 100644 --- a/src/server.h +++ b/src/server.h @@ -2173,7 +2173,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range); unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range); unsigned long zsetLength(const robj *zobj); void zsetConvert(robj *zobj, int encoding); -void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen); +void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen); int zsetScore(robj *zobj, sds member, double *score); unsigned long zslGetRank(zskiplist *zsl, double score, sds o); int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, double *newscore); diff --git a/src/t_hash.c b/src/t_hash.c index ea0606fb0..2720fdbc7 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -39,17 +39,22 @@ * as their string length can be queried in constant time. */ void hashTypeTryConversion(robj *o, robj **argv, int start, int end) { int i; + size_t sum = 0; if (o->encoding != OBJ_ENCODING_ZIPLIST) return; for (i = start; i <= end; i++) { - if (sdsEncodedObject(argv[i]) && - sdslen(argv[i]->ptr) > server.hash_max_ziplist_value) - { + if (!sdsEncodedObject(argv[i])) + continue; + size_t len = sdslen(argv[i]->ptr); + if (len > server.hash_max_ziplist_value) { hashTypeConvert(o, OBJ_ENCODING_HT); - break; + return; } + sum += len; } + if (!ziplistSafeToAdd(o->ptr, sum)) + hashTypeConvert(o, OBJ_ENCODING_HT); } /* Get the value from a ziplist encoded hash, identified by field. diff --git a/src/t_list.c b/src/t_list.c index f8ca27458..66c9e3c9d 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -29,6 +29,8 @@ #include "server.h" +#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024) + /*----------------------------------------------------------------------------- * List API *----------------------------------------------------------------------------*/ @@ -224,6 +226,13 @@ robj *listTypeDup(robj *o) { void pushGenericCommand(client *c, int where, int xx) { int j; + for (j = 2; j < c->argc; j++) { + if (sdslen(c->argv[j]->ptr) > LIST_MAX_ITEM_SIZE) { + addReplyError(c, "Element too large"); + return; + } + } + robj *lobj = lookupKeyWrite(c->db, c->argv[1]); if (checkType(c,lobj,OBJ_LIST)) return; if (!lobj) { @@ -287,6 +296,11 @@ void linsertCommand(client *c) { return; } + if (sdslen(c->argv[4]->ptr) > LIST_MAX_ITEM_SIZE) { + addReplyError(c, "Element too large"); + return; + } + if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || checkType(c,subject,OBJ_LIST)) return; @@ -354,6 +368,11 @@ void lsetCommand(client *c) { long index; robj *value = c->argv[3]; + if (sdslen(value->ptr) > LIST_MAX_ITEM_SIZE) { + addReplyError(c, "Element too large"); + return; + } + if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK)) return; @@ -576,6 +595,11 @@ void lposCommand(client *c) { int direction = LIST_TAIL; long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */ + if (sdslen(ele->ptr) > LIST_MAX_ITEM_SIZE) { + addReplyError(c, "Element too large"); + return; + } + /* Parse the optional arguments. */ for (int j = 3; j < c->argc; j++) { char *opt = c->argv[j]->ptr; @@ -671,6 +695,11 @@ void lremCommand(client *c) { long toremove; long removed = 0; + if (sdslen(obj->ptr) > LIST_MAX_ITEM_SIZE) { + addReplyError(c, "Element too large"); + return; + } + if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK)) return; diff --git a/src/t_stream.c b/src/t_stream.c index 2c30faa06..574195ee3 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -47,6 +47,12 @@ * setting stream_node_max_bytes to a huge number. */ #define STREAM_LISTPACK_MAX_PRE_ALLOCATE 4096 +/* Don't let listpacks grow too big, even if the user config allows it. + * doing so can lead to an overflow (trying to store more than 32bit length + * into the listpack header), or actually an assertion since lpInsert + * will return NULL. */ +#define STREAM_LISTPACK_MAX_SIZE (1<<30) + void streamFreeCG(streamCG *cg); void streamFreeNACK(streamNACK *na); size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer); @@ -433,8 +439,11 @@ void streamGetEdgeID(stream *s, int first, streamID *edge_id) * * The function returns C_OK if the item was added, this is always true * if the ID was generated by the function. However the function may return - * C_ERR if an ID was given via 'use_id', but adding it failed since the - * current top ID is greater or equal. */ + * C_ERR in several cases: + * 1. If an ID was given via 'use_id', but adding it failed since the + * current top ID is greater or equal. errno will be set to EDOM. + * 2. If a size of a single element or the sum of the elements is too big to + * be stored into the stream. errno will be set to ERANGE. */ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) { /* Generate the new entry ID. */ @@ -448,7 +457,23 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ * or return an error. Automatically generated IDs might * overflow (and wrap-around) when incrementing the sequence part. */ - if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR; + if (streamCompareID(&id,&s->last_id) <= 0) { + errno = EDOM; + return C_ERR; + } + + /* Avoid overflow when trying to add an element to the stream (listpack + * can only host up to 32bit length sttrings, and also a total listpack size + * can't be bigger than 32bit length. */ + size_t totelelen = 0; + for (int64_t i = 0; i < numfields*2; i++) { + sds ele = argv[i]->ptr; + totelelen += sdslen(ele); + } + if (totelelen > STREAM_LISTPACK_MAX_SIZE) { + errno = ERANGE; + return C_ERR; + } /* Add the new entry. */ raxIterator ri; @@ -507,9 +532,10 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_ * if we need to switch to the next one. 'lp' will be set to NULL if * the current node is full. */ if (lp != NULL) { - if (server.stream_node_max_bytes && - lp_bytes >= server.stream_node_max_bytes) - { + size_t node_max_bytes = server.stream_node_max_bytes; + if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE) + node_max_bytes = STREAM_LISTPACK_MAX_SIZE; + if (lp_bytes + totelelen >= node_max_bytes) { lp = NULL; } else if (server.stream_node_max_entries) { unsigned char *lp_ele = lpFirst(lp); @@ -1796,11 +1822,13 @@ void xaddCommand(client *c) { /* Append using the low level function and return the ID. */ streamID id; if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2, - &id, parsed_args.id_given ? &parsed_args.id : NULL) - == C_ERR) + &id, parsed_args.id_given ? &parsed_args.id : NULL) == C_ERR) { - addReplyError(c,"The ID specified in XADD is equal or smaller than the " - "target stream top item"); + if (errno == EDOM) + addReplyError(c,"The ID specified in XADD is equal or smaller than " + "the target stream top item"); + else + addReplyError(c,"Elements are too large to be stored"); return; } addReplyStreamID(c,&id); diff --git a/src/t_zset.c b/src/t_zset.c index 3b9ebd2bd..2abc1b49b 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1242,15 +1242,18 @@ void zsetConvert(robj *zobj, int encoding) { } /* Convert the sorted set object into a ziplist if it is not already a ziplist - * and if the number of elements and the maximum element size is within the - * expected ranges. */ -void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) { + * and if the number of elements and the maximum element size and total elements size + * are within the expected ranges. */ +void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) { if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return; zset *zset = zobj->ptr; if (zset->zsl->length <= server.zset_max_ziplist_entries && - maxelelen <= server.zset_max_ziplist_value) - zsetConvert(zobj,OBJ_ENCODING_ZIPLIST); + maxelelen <= server.zset_max_ziplist_value && + ziplistSafeToAdd(NULL, totelelen)) + { + zsetConvert(zobj,OBJ_ENCODING_ZIPLIST); + } } /* Return (by reference) the score of the specified member of the sorted set @@ -1370,20 +1373,28 @@ int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, dou } return 1; } else if (!xx) { - /* Optimize: check if the element is too large or the list + /* check if the element is too large or the list * becomes too long *before* executing zzlInsert. */ - zobj->ptr = zzlInsert(zobj->ptr,ele,score); - if (zzlLength(zobj->ptr) > server.zset_max_ziplist_entries || - sdslen(ele) > server.zset_max_ziplist_value) + if (zzlLength(zobj->ptr)+1 > server.zset_max_ziplist_entries || + sdslen(ele) > server.zset_max_ziplist_value || + !ziplistSafeToAdd(zobj->ptr, sdslen(ele))) + { zsetConvert(zobj,OBJ_ENCODING_SKIPLIST); - if (newscore) *newscore = score; - *out_flags |= ZADD_OUT_ADDED; - return 1; + } else { + zobj->ptr = zzlInsert(zobj->ptr,ele,score); + if (newscore) *newscore = score; + *out_flags |= ZADD_OUT_ADDED; + return 1; + } } else { *out_flags |= ZADD_OUT_NOP; return 1; } - } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { + } + + /* Note that the above block handling ziplist would have either returned or + * converted the key to skiplist. */ + if (zobj->encoding == OBJ_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; zskiplistNode *znode; dictEntry *de; @@ -2361,7 +2372,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat } } -static int zsetDictGetMaxElementLength(dict *d) { +static size_t zsetDictGetMaxElementLength(dict *d, size_t *totallen) { dictIterator *di; dictEntry *de; size_t maxelelen = 0; @@ -2371,6 +2382,8 @@ static int zsetDictGetMaxElementLength(dict *d) { while((de = dictNext(di)) != NULL) { sds ele = dictGetKey(de); if (sdslen(ele) > maxelelen) maxelelen = sdslen(ele); + if (totallen) + (*totallen) += sdslen(ele); } dictReleaseIterator(di); @@ -2378,7 +2391,7 @@ static int zsetDictGetMaxElementLength(dict *d) { return maxelelen; } -static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { +static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { /* DIFF Algorithm 1: * * We perform the diff by iterating all the elements of the first set, @@ -2426,13 +2439,14 @@ static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t * znode = zslInsert(dstzset->zsl,zval.score,tmp); dictAdd(dstzset->dict,tmp,&znode->score); if (sdslen(tmp) > *maxelelen) *maxelelen = sdslen(tmp); + (*totelelen) += sdslen(tmp); } } zuiClearIterator(&src[0]); } -static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { +static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { /* DIFF Algorithm 2: * * Add all the elements of the first set to the auxiliary set. @@ -2486,7 +2500,7 @@ static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t * /* Using this algorithm, we can't calculate the max element as we go, * we have to iterate through all elements to find the max one after. */ - *maxelelen = zsetDictGetMaxElementLength(dstzset->dict); + *maxelelen = zsetDictGetMaxElementLength(dstzset->dict, totelelen); } static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) { @@ -2523,14 +2537,14 @@ static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) { return (algo_one_work <= algo_two_work) ? 1 : 2; } -static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) { +static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) { /* Skip everything if the smallest input is empty. */ if (zuiLength(&src[0]) > 0) { int diff_algo = zsetChooseDiffAlgorithm(src, setnum); if (diff_algo == 1) { - zdiffAlgorithm1(src, setnum, dstzset, maxelelen); + zdiffAlgorithm1(src, setnum, dstzset, maxelelen, totelelen); } else if (diff_algo == 2) { - zdiffAlgorithm2(src, setnum, dstzset, maxelelen); + zdiffAlgorithm2(src, setnum, dstzset, maxelelen, totelelen); } else if (diff_algo != 0) { serverPanic("Unknown algorithm"); } @@ -2565,7 +2579,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in zsetopsrc *src; zsetopval zval; sds tmp; - size_t maxelelen = 0; + size_t maxelelen = 0, totelelen = 0; robj *dstobj; zset *dstzset; zskiplistNode *znode; @@ -2701,6 +2715,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in tmp = zuiNewSdsFromValue(&zval); znode = zslInsert(dstzset->zsl,score,tmp); dictAdd(dstzset->dict,tmp,&znode->score); + totelelen += sdslen(tmp); if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp); } } @@ -2737,6 +2752,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in /* Remember the longest single element encountered, * to understand if it's possible to convert to ziplist * at the end. */ + totelelen += sdslen(tmp); if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp); /* Update the element with its initial score. */ dictSetKey(accumulator, de, tmp); @@ -2771,14 +2787,14 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in dictReleaseIterator(di); dictRelease(accumulator); } else if (op == SET_OP_DIFF) { - zdiff(src, setnum, dstzset, &maxelelen); + zdiff(src, setnum, dstzset, &maxelelen, &totelelen); } else { serverPanic("Unknown operator"); } if (dstkey) { if (dstzset->zsl->length) { - zsetConvertToZiplistIfNeeded(dstobj, maxelelen); + zsetConvertToZiplistIfNeeded(dstobj, maxelelen, totelelen); setKey(c, c->db, dstkey, dstobj); addReplyLongLong(c, zsetLength(dstobj)); notifyKeyspaceEvent(NOTIFY_ZSET, diff --git a/src/ziplist.c b/src/ziplist.c index aae86c1f2..fdc1bb9e1 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -267,6 +267,17 @@ ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \ } +/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in + * zlbytes*/ +#define ZIPLIST_MAX_SAFETY_SIZE (1<<30) +int ziplistSafeToAdd(unsigned char* zl, size_t add) { + size_t len = zl? ziplistBlobLen(zl): 0; + if (len + add > ZIPLIST_MAX_SAFETY_SIZE) + return 0; + return 1; +} + + /* We use this function to receive information about a ziplist entry. * Note that this is not how the data is actually encoded, is just what we * get filled by a function in order to operate more easily. */ @@ -709,7 +720,8 @@ unsigned char *ziplistNew(void) { } /* Resize the ziplist. */ -unsigned char *ziplistResize(unsigned char *zl, unsigned int len) { +unsigned char *ziplistResize(unsigned char *zl, size_t len) { + assert(len < UINT32_MAX); zl = zrealloc(zl,len); ZIPLIST_BYTES(zl) = intrev32ifbe(len); zl[len-1] = ZIP_END; @@ -1070,6 +1082,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) { /* Combined zl length should be limited within UINT16_MAX */ zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX; + /* larger values can't be stored into ZIPLIST_BYTES */ + assert(zlbytes < UINT32_MAX); + /* Save offset positions before we start ripping memory apart. */ size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first)); size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second)); diff --git a/src/ziplist.h b/src/ziplist.h index 9e7997ad8..569e1259d 100644 --- a/src/ziplist.h +++ b/src/ziplist.h @@ -65,6 +65,7 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep, void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val); void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals); unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals); +int ziplistSafeToAdd(unsigned char* zl, size_t add); #ifdef REDIS_TEST int ziplistTest(int argc, char *argv[], int accurate); diff --git a/tests/unit/violations.tcl b/tests/unit/violations.tcl new file mode 100644 index 000000000..1d3140c52 --- /dev/null +++ b/tests/unit/violations.tcl @@ -0,0 +1,156 @@ +# These tests consume massive amounts of memory, and are not +# suitable to be executed as part of the normal test suite +set ::str500 [string repeat x 500000000] ;# 500mb + +# Utility function to write big argument into redis client connection +proc write_big_bulk {size} { + r write "\$$size\r\n" + while {$size >= 500000000} { + r write $::str500 + incr size -500000000 + } + if {$size > 0} { + r write [string repeat x $size] + } + r write "\r\n" +} + +# One XADD with one huge 5GB field +# Expected to fail resulting in an empty stream +start_server [list overrides [list save ""] ] { + test {XADD one huge field} { + r config set proto-max-bulk-len 10000000000 ;#10gb + r config set client-query-buffer-limit 10000000000 ;#10gb + r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n" + r write "\$1\r\nA\r\n" + write_big_bulk 5000000000 ;#5gb + r flush + catch {r read} err + assert_match {*too large*} $err + r xlen S1 + } {0} +} + +# One XADD with one huge (exactly nearly) 4GB field +# This uncovers the overflow in lpEncodeGetType +# Expected to fail resulting in an empty stream +start_server [list overrides [list save ""] ] { + test {XADD one huge field - 1} { + r config set proto-max-bulk-len 10000000000 ;#10gb + r config set client-query-buffer-limit 10000000000 ;#10gb + r write "*5\r\n\$4\r\nXADD\r\n\$2\r\nS1\r\n\$1\r\n*\r\n" + r write "\$1\r\nA\r\n" + write_big_bulk 4294967295 ;#4gb-1 + r flush + catch {r read} err + assert_match {*too large*} $err + r xlen S1 + } {0} +} + +# Gradually add big stream fields using repeated XADD calls +start_server [list overrides [list save ""] ] { + test {several XADD big fields} { + r config set stream-node-max-bytes 0 + for {set j 0} {$j<10} {incr j} { + r xadd stream * 1 $::str500 2 $::str500 + } + r ping + r xlen stream + } {10} +} + +# Add over 4GB to a single stream listpack (one XADD command) +# Expected to fail resulting in an empty stream +start_server [list overrides [list save ""] ] { + test {single XADD big fields} { + r write "*23\r\n\$4\r\nXADD\r\n\$1\r\nS\r\n\$1\r\n*\r\n" + for {set j 0} {$j<10} {incr j} { + r write "\$1\r\n$j\r\n" + write_big_bulk 500000000 ;#500mb + } + r flush + catch {r read} err + assert_match {*too large*} $err + r xlen S + } {0} +} + +# Gradually add big hash fields using repeated HSET calls +# This reproduces the overflow in the call to ziplistResize +# Object will be converted to hashtable encoding +start_server [list overrides [list save ""] ] { + r config set hash-max-ziplist-value 1000000000 ;#1gb + test {hash with many big fields} { + for {set j 0} {$j<10} {incr j} { + r hset h $j $::str500 + } + r object encoding h + } {hashtable} +} + +# Add over 4GB to a single hash field (one HSET command) +# Object will be converted to hashtable encoding +start_server [list overrides [list save ""] ] { + test {hash with one huge field} { + catch {r config set hash-max-ziplist-value 10000000000} ;#10gb + r config set proto-max-bulk-len 10000000000 ;#10gb + r config set client-query-buffer-limit 10000000000 ;#10gb + r write "*4\r\n\$4\r\nHSET\r\n\$2\r\nH1\r\n" + r write "\$1\r\nA\r\n" + write_big_bulk 5000000000 ;#5gb + r flush + r read + r object encoding H1 + } {hashtable} +} + +# Add over 4GB to a single list member (one LPUSH command) +# Currently unsupported, and expected to fail rather than being truncated +# Expected to fail resulting in a non-existing list +start_server [list overrides [list save ""] ] { + test {list with one huge field} { + r config set proto-max-bulk-len 10000000000 ;#10gb + r config set client-query-buffer-limit 10000000000 ;#10gb + r write "*3\r\n\$5\r\nLPUSH\r\n\$2\r\nL1\r\n" + write_big_bulk 5000000000 ;#5gb + r flush + catch {r read} err + assert_match {*too large*} $err + r exists L1 + } {0} +} + +# SORT which attempts to store an element larger than 4GB into a list. +# Currently unsupported and results in an assertion instead of truncation +start_server [list overrides [list save ""] ] { + test {SORT adds huge field to list} { + r config set proto-max-bulk-len 10000000000 ;#10gb + r config set client-query-buffer-limit 10000000000 ;#10gb + r write "*3\r\n\$3\r\nSET\r\n\$2\r\nS1\r\n" + write_big_bulk 5000000000 ;#5gb + r flush + r read + assert_equal [r strlen S1] 5000000000 + r set S2 asdf + r sadd myset 1 2 + r mset D1 1 D2 2 + catch {r sort myset by D* get S* store mylist} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +# SORT which stores an integer encoded element into a list. +# Just for coverage, no news here. +start_server [list overrides [list save ""] ] { + test {SORT adds integer field to list} { + r set S1 asdf + r set S2 123 ;# integer encoded + assert_encoding "int" S2 + r sadd myset 1 2 + r mset D1 1 D2 2 + r sort myset by D* get S* store mylist + r llen mylist + } {2} +} From 1b3eace3561e15c9dd85ca9ba29581772fa0540b Mon Sep 17 00:00:00 2001 From: YiyuanGUO Date: Wed, 29 Sep 2021 10:20:35 +0300 Subject: [PATCH 54/79] Fix integer overflow in _sdsMakeRoomFor (CVE-2021-41099) --- src/sds.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sds.c b/src/sds.c index 8d1c2dbd0..65fb3c717 100644 --- a/src/sds.c +++ b/src/sds.c @@ -233,7 +233,7 @@ void sdsclear(sds s) { sds sdsMakeRoomFor(sds s, size_t addlen) { void *sh, *newsh; size_t avail = sdsavail(s); - size_t len, newlen; + size_t len, newlen, reqlen; char type, oldtype = s[-1] & SDS_TYPE_MASK; int hdrlen; size_t usable; @@ -243,7 +243,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { len = sdslen(s); sh = (char*)s-sdsHdrSize(oldtype); - newlen = (len+addlen); + reqlen = newlen = (len+addlen); assert(newlen > len); /* Catch size_t overflow */ if (newlen < SDS_MAX_PREALLOC) newlen *= 2; @@ -258,7 +258,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { if (type == SDS_TYPE_5) type = SDS_TYPE_8; hdrlen = sdsHdrSize(type); - assert(hdrlen + newlen + 1 > len); /* Catch size_t overflow */ + assert(hdrlen + newlen + 1 > reqlen); /* Catch size_t overflow */ if (oldtype==type) { newsh = s_realloc_usable(sh, hdrlen+newlen+1, &usable); if (newsh == NULL) return NULL; From 9d66fb2d8b984052ca5fc988fdab0ae4fee96b77 Mon Sep 17 00:00:00 2001 From: Binbin Date: Mon, 2 Aug 2021 00:32:24 +0800 Subject: [PATCH 55/79] GEO* STORE with empty src key delete the dest key and return 0, not empty array (#9271) With an empty src key, we need to deal with two situations: 1. non-STORE: We should return emptyarray. 2. STORE: Try to delete the store key and return 0. This applies to both GEOSEARCHSTORE (new to v6.2), and also GEORADIUS STORE (which was broken since forever) This pr try to fix #9261. i.e. both STORE variants would have behaved like the non-STORE variants when the source key was missing, returning an empty array and not deleting the destination key, instead of returning 0, and deleting the destination key. Also add more tests for some commands. - GEORADIUS: wrong type src key, non existing src key, empty search, store with non existing src key, store with empty search - GEORADIUSBYMEMBER: wrong type src key, non existing src key, non existing member, store with non existing src key - GEOSEARCH: wrong type src key, non existing src key, empty search, frommember with non existing member - GEOSEARCHSTORE: wrong type key, non existing src key, fromlonlat with empty search, frommember with non existing member Co-authored-by: Oran Agra (cherry picked from commit 86555ae0f7cc45abac7f758d72bf456e90793b46) --- src/geo.c | 40 ++++++++++++++++++++++---- tests/unit/geo.tcl | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 6 deletions(-) diff --git a/src/geo.c b/src/geo.c index 893f78a7e..c3f6c922c 100644 --- a/src/geo.c +++ b/src/geo.c @@ -509,7 +509,7 @@ void geoaddCommand(client *c) { * [COUNT count [ANY]] [STORE key] [STOREDIST key] * GEORADIUSBYMEMBER key member radius unit ... options ... * GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit] - * [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] + * [BYBOX width height unit] [WITHCOORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit] * [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] [STOREDIST] * */ @@ -518,21 +518,24 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */ /* Look up the requested zset */ - robj *zobj = NULL; - if ((zobj = lookupKeyReadOrReply(c, c->argv[srcKeyIndex], shared.emptyarray)) == NULL || - checkType(c, zobj, OBJ_ZSET)) { - return; - } + robj *zobj = lookupKeyRead(c->db, c->argv[srcKeyIndex]); + if (checkType(c, zobj, OBJ_ZSET)) return; /* Find long/lat to use for radius or box search based on inquiry type */ int base_args; GeoShape shape = {0}; if (flags & RADIUS_COORDS) { + /* GEORADIUS or GEORADIUS_RO */ base_args = 6; shape.type = CIRCULAR_TYPE; if (extractLongLatOrReply(c, c->argv + 2, shape.xy) == C_ERR) return; if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return; + } else if ((flags & RADIUS_MEMBER) && !zobj) { + /* We don't have a source key, but we need to proceed with argument + * parsing, so we know which reply to use depending on the STORE flag. */ + base_args = 5; } else if (flags & RADIUS_MEMBER) { + /* GEORADIUSBYMEMBER or GEORADIUSBYMEMBER_RO */ base_args = 5; shape.type = CIRCULAR_TYPE; robj *member = c->argv[2]; @@ -542,6 +545,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { } if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return; } else if (flags & GEOSEARCH) { + /* GEOSEARCH or GEOSEARCHSTORE */ base_args = 2; if (flags & GEOSEARCHSTORE) { base_args = 3; @@ -608,6 +612,13 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { flags & GEOSEARCH && !fromloc) { + /* No source key, proceed with argument parsing and return an error when done. */ + if (zobj == NULL) { + frommember = 1; + i++; + continue; + } + if (longLatFromMember(zobj, c->argv[base_args+i+1], shape.xy) == C_ERR) { addReplyError(c, "could not decode requested zset member"); return; @@ -676,6 +687,23 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) { return; } + /* Return ASAP when src key does not exist. */ + if (zobj == NULL) { + if (storekey) { + /* store key is not NULL, try to delete it and return 0. */ + if (dbDelete(c->db, storekey)) { + signalModifiedKey(c, c->db, storekey); + notifyKeyspaceEvent(NOTIFY_GENERIC, "del", storekey, c->db->id); + server.dirty++; + } + addReply(c, shared.czero); + } else { + /* Otherwise we return an empty array. */ + addReply(c, shared.emptyarray); + } + return; + } + /* COUNT without ordering does not make much sense (we need to * sort in order to return the closest N entries), * force ASC ordering if COUNT was specified but no sorting was diff --git a/tests/unit/geo.tcl b/tests/unit/geo.tcl index a51d1dc5c..3aa8f4d56 100644 --- a/tests/unit/geo.tcl +++ b/tests/unit/geo.tcl @@ -71,6 +71,34 @@ proc pointInRectangle {width_km height_km lon lat search_lon search_lat error} { return true } +proc verify_geo_edge_response_bylonlat {expected_response expected_store_response} { + catch {r georadius src{t} 1 1 1 km} response + assert_match $expected_response $response + + catch {r georadius src{t} 1 1 1 km store dest{t}} response + assert_match $expected_store_response $response + + catch {r geosearch src{t} fromlonlat 0 0 byradius 1 km} response + assert_match $expected_response $response + + catch {r geosearchstore dest{t} src{t} fromlonlat 0 0 byradius 1 km} response + assert_match $expected_store_response $response +} + +proc verify_geo_edge_response_bymember {expected_response expected_store_response} { + catch {r georadiusbymember src{t} member 1 km} response + assert_match $expected_response $response + + catch {r georadiusbymember src{t} member 1 km store dest{t}} response + assert_match $expected_store_response $response + + catch {r geosearch src{t} frommember member bybox 1 1 km} response + assert_match $expected_response $response + + catch {r geosearchstore dest{t} src{t} frommember member bybox 1 1 m} response + assert_match $expected_store_response $response +} + # The following list represents sets of random seed, search position # and radius that caused bugs in the past. It is used by the randomized # test later as a starting point. When the regression vectors are scanned @@ -95,6 +123,34 @@ set regression_vectors { set rv_idx 0 start_server {tags {"geo"}} { + test {GEO with wrong type src key} { + r set src{t} wrong_type + + verify_geo_edge_response_bylonlat "WRONGTYPE*" "WRONGTYPE*" + verify_geo_edge_response_bymember "WRONGTYPE*" "WRONGTYPE*" + } + + test {GEO with non existing src key} { + r del src{t} + + verify_geo_edge_response_bylonlat {} 0 + verify_geo_edge_response_bymember {} 0 + } + + test {GEO BYLONLAT with empty search} { + r del src{t} + r geoadd src{t} 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" + + verify_geo_edge_response_bylonlat {} 0 + } + + test {GEO BYMEMBER with non existing member} { + r del src{t} + r geoadd src{t} 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" + + verify_geo_edge_response_bymember "ERR*" "ERR*" + } + test {GEOADD create} { r geoadd nyc -73.9454966 40.747533 "lic market" } {1} @@ -357,6 +413,21 @@ start_server {tags {"geo"}} { assert_equal [r zrange points 0 -1] [r zrange points2 0 -1] } + test {GEORADIUSBYMEMBER STORE/STOREDIST option: plain usage} { + r del points{t} + r geoadd points{t} 13.361389 38.115556 "Palermo" 15.087269 37.502669 "Catania" + + r georadiusbymember points{t} Palermo 500 km store points2{t} + assert_equal {Palermo Catania} [r zrange points2{t} 0 -1] + + r georadiusbymember points{t} Catania 500 km storedist points2{t} + assert_equal {Catania Palermo} [r zrange points2{t} 0 -1] + + set res [r zrange points2{t} 0 -1 withscores] + assert {[lindex $res 1] < 1} + assert {[lindex $res 3] > 166} + } + test {GEOSEARCHSTORE STORE option: plain usage} { r geosearchstore points2 points fromlonlat 13.361389 38.115556 byradius 500 km assert_equal [r zrange points 0 -1] [r zrange points2 0 -1] From a3cb1bb0dbd4fbaf05136764c05de6c23264b904 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Thu, 6 May 2021 23:34:45 +0800 Subject: [PATCH 56/79] redis-cli when SELECT fails, we should reset dbnum to 0 (#8898) when SELECT fails, we should reset dbnum to 0, so the prompt will not display incorrectly. Additionally when SELECT and HELLO fail, we output message to inform it. Add config.input_dbnum which means the dbnum about to select. And config.dbnum means currently selected dbnum. When users succeed to select db, config.dbnum and config.input_dbnum will be the same. When users select db failed, config.input_dbnum will be kept. Next time if users auth success, config.input_dbnum will be automatically selected. When reconnect, we should select the origin dbnum. Co-authored-by: Oran Agra (cherry picked from commit 6b475989984bb28499327e33cc79315d6264bc06) --- src/redis-cli.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 3160ada46..3b5869433 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -207,7 +207,8 @@ static struct config { cliSSLconfig sslconfig; long repeat; long interval; - int dbnum; + int dbnum; /* db num currently selected */ + int input_dbnum; /* db num user input */ int interactive; int shutdown; int monitor_mode; @@ -445,7 +446,7 @@ static void parseRedisUri(const char *uri) { if (curr == end) return; /* Extract database number. */ - config.dbnum = atoi(curr); + config.input_dbnum = atoi(curr); } static uint64_t dictSdsHash(const void *key) { @@ -801,15 +802,21 @@ static int cliAuth(redisContext *ctx, char *user, char *auth) { return REDIS_ERR; } -/* Send SELECT dbnum to the server */ +/* Send SELECT input_dbnum to the server */ static int cliSelect(void) { redisReply *reply; - if (config.dbnum == 0) return REDIS_OK; + if (config.input_dbnum == config.dbnum) return REDIS_OK; - reply = redisCommand(context,"SELECT %d",config.dbnum); + reply = redisCommand(context,"SELECT %d",config.input_dbnum); if (reply != NULL) { int result = REDIS_OK; - if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr,"SELECT %d failed: %s\n",config.input_dbnum,reply->str); + } else { + config.dbnum = config.input_dbnum; + cliRefreshPrompt(); + } freeReplyObject(reply); return result; } @@ -824,7 +831,10 @@ static int cliSwitchProto(void) { reply = redisCommand(context,"HELLO 3"); if (reply != NULL) { int result = REDIS_OK; - if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr,"HELLO 3 failed: %s\n",reply->str); + } freeReplyObject(reply); return result; } @@ -1434,7 +1444,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) { if (!strcasecmp(command,"select") && argc == 2 && config.last_cmd_type != REDIS_REPLY_ERROR) { - config.dbnum = atoi(argv[1]); + config.input_dbnum = config.dbnum = atoi(argv[1]); cliRefreshPrompt(); } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) { cliSelect(); @@ -1446,15 +1456,17 @@ static int cliSendCommand(int argc, char **argv, long repeat) { cliRefreshPrompt(); } else if (!strcasecmp(command,"exec") && argc == 1 && config.in_multi) { config.in_multi = 0; - if (config.last_cmd_type == REDIS_REPLY_ERROR) { - config.dbnum = config.pre_multi_dbnum; + if (config.last_cmd_type == REDIS_REPLY_ERROR || + config.last_cmd_type == REDIS_REPLY_NIL) + { + config.input_dbnum = config.dbnum = config.pre_multi_dbnum; } cliRefreshPrompt(); } else if (!strcasecmp(command,"discard") && argc == 1 && config.last_cmd_type != REDIS_REPLY_ERROR) { config.in_multi = 0; - config.dbnum = config.pre_multi_dbnum; + config.input_dbnum = config.dbnum = config.pre_multi_dbnum; cliRefreshPrompt(); } } @@ -1541,7 +1553,7 @@ static int parseOptions(int argc, char **argv) { double seconds = atof(argv[++i]); config.interval = seconds*1000000; } else if (!strcmp(argv[i],"-n") && !lastarg) { - config.dbnum = atoi(argv[++i]); + config.input_dbnum = atoi(argv[++i]); } else if (!strcmp(argv[i], "--no-auth-warning")) { config.no_auth_warning = 1; } else if (!strcmp(argv[i], "--askpass")) { @@ -8209,6 +8221,7 @@ int main(int argc, char **argv) { config.repeat = 1; config.interval = 0; config.dbnum = 0; + config.input_dbnum = 0; config.interactive = 0; config.shutdown = 0; config.monitor_mode = 0; From e87d855ab205ce2e39d97405542ef472439eae23 Mon Sep 17 00:00:00 2001 From: Binbin Date: Tue, 18 May 2021 15:09:45 +0800 Subject: [PATCH 57/79] redis-cli: Sleep for a while in each cliConnect when we got connect error in cluster mode. (#8884) There's an infinite loop when redis-cli fails to connect in cluster mode. This commit adds a 1 second sleep to prevent flooding the console with errors. It also adds a specific error print in a few places that could have error without printing anything. Co-authored-by: Oran Agra (cherry picked from commit 8351a10b959364cff9fc026188ebc9c653ef230a) --- src/redis-cli.c | 80 +++++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 3b5869433..cc1c10193 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -793,13 +793,19 @@ static int cliAuth(redisContext *ctx, char *user, char *auth) { reply = redisCommand(ctx,"AUTH %s",auth); else reply = redisCommand(ctx,"AUTH %s %s",user,auth); - if (reply != NULL) { - if (reply->type == REDIS_REPLY_ERROR) - fprintf(stderr,"Warning: AUTH failed\n"); - freeReplyObject(reply); - return REDIS_OK; + + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + return REDIS_ERR; } - return REDIS_ERR; + + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr, "AUTH failed: %s\n", reply->str); + } + freeReplyObject(reply); + return result; } /* Send SELECT input_dbnum to the server */ @@ -808,19 +814,21 @@ static int cliSelect(void) { if (config.input_dbnum == config.dbnum) return REDIS_OK; reply = redisCommand(context,"SELECT %d",config.input_dbnum); - if (reply != NULL) { - int result = REDIS_OK; - if (reply->type == REDIS_REPLY_ERROR) { - result = REDIS_ERR; - fprintf(stderr,"SELECT %d failed: %s\n",config.input_dbnum,reply->str); - } else { - config.dbnum = config.input_dbnum; - cliRefreshPrompt(); - } - freeReplyObject(reply); - return result; + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + return REDIS_ERR; } - return REDIS_ERR; + + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr,"SELECT %d failed: %s\n",config.input_dbnum,reply->str); + } else { + config.dbnum = config.input_dbnum; + cliRefreshPrompt(); + } + freeReplyObject(reply); + return result; } /* Select RESP3 mode if redis-cli was started with the -3 option. */ @@ -829,16 +837,18 @@ static int cliSwitchProto(void) { if (config.resp3 == 0) return REDIS_OK; reply = redisCommand(context,"HELLO 3"); - if (reply != NULL) { - int result = REDIS_OK; - if (reply->type == REDIS_REPLY_ERROR) { - result = REDIS_ERR; - fprintf(stderr,"HELLO 3 failed: %s\n",reply->str); - } - freeReplyObject(reply); - return result; + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + return REDIS_ERR; } - return REDIS_ERR; + + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr,"HELLO 3 failed: %s\n",reply->str); + } + freeReplyObject(reply); + return result; } /* Connect to the server. It is possible to pass certain flags to the function: @@ -875,12 +885,15 @@ static int cliConnect(int flags) { if (context->err) { if (!(flags & CC_QUIET)) { fprintf(stderr,"Could not connect to Redis at "); - if (config.hostsocket == NULL) - fprintf(stderr,"%s:%d: %s\n", + if (config.hostsocket == NULL || + (config.cluster_mode && config.cluster_reissue_command)) + { + fprintf(stderr, "%s:%d: %s\n", config.hostip,config.hostport,context->errstr); - else + } else { fprintf(stderr,"%s: %s\n", config.hostsocket,context->errstr); + } } redisFree(context); context = NULL; @@ -1472,7 +1485,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) { } if (config.cluster_reissue_command){ /* If we need to reissue the command, break to prevent a - further 'repeat' number of dud interations */ + further 'repeat' number of dud interactions */ break; } if (config.interval) usleep(config.interval); @@ -2019,9 +2032,12 @@ static int issueCommandRepeat(int argc, char **argv, long repeat) { return REDIS_ERR; } } + /* Issue the command again if we got redirected in cluster mode */ if (config.cluster_mode && config.cluster_reissue_command) { - cliConnect(CC_FORCE); + /* If cliConnect fails, sleep for a while and try again. */ + if (cliConnect(CC_FORCE) != REDIS_OK) + sleep(1); } else { break; } From 68e3acf11361646f3c5f8d511607030682112b34 Mon Sep 17 00:00:00 2001 From: Huang Zhw Date: Mon, 2 Aug 2021 19:59:08 +0800 Subject: [PATCH 58/79] When redis-cli received ASK, it didn't handle it (#8930) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When redis-cli received ASK, it used string matching wrong and didn't handle it. When we access a slot which is in migrating state, it maybe return ASK. After redirect to the new node, we need send ASKING command before retry the command. In this PR after redis-cli receives ASK, we send a ASKING command before send the origin command after reconnecting. Other changes: * Make redis-cli -u and -c (unix socket and cluster mode) incompatible with one another. * When send command fails, we avoid the 2nd reconnect retry and just print the error info. Users will decide how to do next. See #9277. * Add a test faking two redis nodes in TCL to just send ASK and OK in redis protocol to test ASK behavior. Co-authored-by: Viktor Söderqvist Co-authored-by: Oran Agra (cherry picked from commit cf61ad14cc45787e57d9af3f28f41462ac0f2aa2) --- src/redis-cli.c | 63 +++++++++++++++++++++++++------ tests/helpers/fake_redis_node.tcl | 58 ++++++++++++++++++++++++++++ tests/integration/redis-cli.tcl | 31 ++++++++++++--- 3 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 tests/helpers/fake_redis_node.tcl diff --git a/src/redis-cli.c b/src/redis-cli.c index cc1c10193..ace378589 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -220,6 +220,7 @@ static struct config { long long lru_test_sample_size; int cluster_mode; int cluster_reissue_command; + int cluster_send_asking; int slave_mode; int pipe_mode; int pipe_timeout; @@ -924,6 +925,29 @@ static int cliConnect(int flags) { return REDIS_OK; } +/* In cluster, if server replies ASK, we will redirect to a different node. + * Before sending the real command, we need to send ASKING command first. */ +static int cliSendAsking() { + redisReply *reply; + + config.cluster_send_asking = 0; + if (context == NULL) { + return REDIS_ERR; + } + reply = redisCommand(context,"ASKING"); + if (reply == NULL) { + fprintf(stderr, "\nI/O error\n"); + return REDIS_ERR; + } + int result = REDIS_OK; + if (reply->type == REDIS_REPLY_ERROR) { + result = REDIS_ERR; + fprintf(stderr,"ASKING failed: %s\n",reply->str); + } + freeReplyObject(reply); + return result; +} + static void cliPrintContextError(void) { if (context == NULL) return; fprintf(stderr,"Error: %s\n",context->errstr); @@ -1299,7 +1323,7 @@ static int cliReadReply(int output_raw_strings) { /* Check if we need to connect to a different node and reissue the * request. */ if (config.cluster_mode && reply->type == REDIS_REPLY_ERROR && - (!strncmp(reply->str,"MOVED",5) || !strcmp(reply->str,"ASK"))) + (!strncmp(reply->str,"MOVED ",6) || !strncmp(reply->str,"ASK ",4))) { char *p = reply->str, *s; int slot; @@ -1323,6 +1347,9 @@ static int cliReadReply(int output_raw_strings) { printf("-> Redirected to slot [%d] located at %s:%d\n", slot, config.hostip, config.hostport); config.cluster_reissue_command = 1; + if (!strncmp(reply->str,"ASK ",4)) { + config.cluster_send_asking = 1; + } cliRefreshPrompt(); } else if (!config.interactive && config.set_errcode && reply->type == REDIS_REPLY_ERROR) @@ -1804,6 +1831,11 @@ static int parseOptions(int argc, char **argv) { } } + if (config.hostsocket && config.cluster_mode) { + fprintf(stderr,"Options -c and -s are mutually exclusive.\n"); + exit(1); + } + /* --ldb requires --eval. */ if (config.eval_ldb && config.eval == NULL) { fprintf(stderr,"Options --ldb and --ldb-sync-mode require --eval.\n"); @@ -2021,26 +2053,32 @@ static sds *getSdsArrayFromArgv(int argc, char **argv, int quoted) { static int issueCommandRepeat(int argc, char **argv, long repeat) { while (1) { + if (config.cluster_reissue_command || context == NULL || + context->err == REDIS_ERR_IO || context->err == REDIS_ERR_EOF) + { + if (cliConnect(CC_FORCE) != REDIS_OK) { + cliPrintContextError(); + config.cluster_reissue_command = 0; + return REDIS_ERR; + } + } config.cluster_reissue_command = 0; - if (cliSendCommand(argc,argv,repeat) != REDIS_OK) { - cliConnect(CC_FORCE); - - /* If we still cannot send the command print error. - * We'll try to reconnect the next time. */ - if (cliSendCommand(argc,argv,repeat) != REDIS_OK) { + if (config.cluster_send_asking) { + if (cliSendAsking() != REDIS_OK) { cliPrintContextError(); return REDIS_ERR; } } + if (cliSendCommand(argc,argv,repeat) != REDIS_OK) { + cliPrintContextError(); + return REDIS_ERR; + } /* Issue the command again if we got redirected in cluster mode */ if (config.cluster_mode && config.cluster_reissue_command) { - /* If cliConnect fails, sleep for a while and try again. */ - if (cliConnect(CC_FORCE) != REDIS_OK) - sleep(1); - } else { - break; + continue; } + break; } return REDIS_OK; } @@ -8248,6 +8286,7 @@ int main(int argc, char **argv) { config.lru_test_mode = 0; config.lru_test_sample_size = 0; config.cluster_mode = 0; + config.cluster_send_asking = 0; config.slave_mode = 0; config.getrdb_mode = 0; config.stat_mode = 0; diff --git a/tests/helpers/fake_redis_node.tcl b/tests/helpers/fake_redis_node.tcl new file mode 100644 index 000000000..a12d87fed --- /dev/null +++ b/tests/helpers/fake_redis_node.tcl @@ -0,0 +1,58 @@ +# A fake Redis node for replaying predefined/expected traffic with a client. +# +# Usage: tclsh fake_redis_node.tcl PORT COMMAND REPLY [ COMMAND REPLY [ ... ] ] +# +# Commands are given as space-separated strings, e.g. "GET foo", and replies as +# RESP-encoded replies minus the trailing \r\n, e.g. "+OK". + +set port [lindex $argv 0]; +set expected_traffic [lrange $argv 1 end]; + +# Reads and parses a command from a socket and returns it as a space-separated +# string, e.g. "set foo bar". +proc read_command {sock} { + set char [read $sock 1] + switch $char { + * { + set numargs [gets $sock] + set result {} + for {set i 0} {$i<$numargs} {incr i} { + read $sock 1; # dollar sign + set len [gets $sock] + set str [read $sock $len] + gets $sock; # trailing \r\n + lappend result $str + } + return $result + } + {} { + # EOF + return {} + } + default { + # Non-RESP command + set rest [gets $sock] + return "$char$rest" + } + } +} + +proc accept {sock host port} { + global expected_traffic + foreach {expect_cmd reply} $expected_traffic { + if {[eof $sock]} {break} + set cmd [read_command $sock] + if {[string equal -nocase $cmd $expect_cmd]} { + puts $sock $reply + flush $sock + } else { + puts $sock "-ERR unexpected command $cmd" + break + } + } + close $sock +} + +socket -server accept $port +after 5000 set done timeout +vwait done diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index d58229bb4..013f76ebf 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -64,8 +64,8 @@ start_server {tags {"cli"}} { set _ $tmp } - proc _run_cli {opts args} { - set cmd [rediscli [srv host] [srv port] [list -n 9 {*}$args]] + proc _run_cli {host port db opts args} { + set cmd [rediscli $host $port [list -n $db {*}$args]] foreach {key value} $opts { if {$key eq "pipe"} { set cmd "sh -c \"$value | $cmd\"" @@ -84,15 +84,19 @@ start_server {tags {"cli"}} { } proc run_cli {args} { - _run_cli {} {*}$args + _run_cli [srv host] [srv port] 9 {} {*}$args } proc run_cli_with_input_pipe {cmd args} { - _run_cli [list pipe $cmd] -x {*}$args + _run_cli [srv host] [srv port] 9 [list pipe $cmd] -x {*}$args } proc run_cli_with_input_file {path args} { - _run_cli [list path $path] -x {*}$args + _run_cli [srv host] [srv port] 9 [list path $path] -x {*}$args + } + + proc run_cli_host_port_db {host port db args} { + _run_cli $host $port $db {} {*}$args } proc test_nontty_cli {name code} { @@ -207,6 +211,23 @@ start_server {tags {"cli"}} { assert_equal "foo\nbar" [run_cli lrange list 0 -1] } + test_nontty_cli "ASK redirect test" { + # Set up two fake Redis nodes. + set tclsh [info nameofexecutable] + set script "tests/helpers/fake_redis_node.tcl" + set port1 [find_available_port $::baseport $::portcount] + set port2 [find_available_port $::baseport $::portcount] + set p1 [exec $tclsh $script $port1 \ + "SET foo bar" "-ASK 12182 127.0.0.1:$port2" &] + set p2 [exec $tclsh $script $port2 \ + "ASKING" "+OK" \ + "SET foo bar" "+OK" &] + # Sleep to make sure both fake nodes have started listening + after 100 + # Run the cli + assert_equal "OK" [run_cli_host_port_db "127.0.0.1" $port1 0 -c SET foo bar] + } + test_nontty_cli "Quoted input arguments" { r set "\x00\x00" "value" assert_equal "value" [run_cli --quoted-input get {"\x00\x00"}] From 667d95f025bf193a59d2582dd09e3b1fd2261a2e Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Tue, 3 Aug 2021 23:19:03 +0300 Subject: [PATCH 59/79] Skip new redis-cli ASK test in TLS mode (#9312) (cherry picked from commit 52df350fe59d73e6a1a4a5fb3c2b91d5c62f5a76) --- tests/integration/redis-cli.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 013f76ebf..3ac36c2dd 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -211,6 +211,7 @@ start_server {tags {"cli"}} { assert_equal "foo\nbar" [run_cli lrange list 0 -1] } +if {!$::tls} { ;# fake_redis_node doesn't support TLS test_nontty_cli "ASK redirect test" { # Set up two fake Redis nodes. set tclsh [info nameofexecutable] @@ -227,6 +228,7 @@ start_server {tags {"cli"}} { # Run the cli assert_equal "OK" [run_cli_host_port_db "127.0.0.1" $port1 0 -c SET foo bar] } +} test_nontty_cli "Quoted input arguments" { r set "\x00\x00" "value" From 39ce98163a8e455381ca1cfa57a2da93869d3eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20S=C3=B6derqvist?= Date: Thu, 5 Aug 2021 07:20:30 +0200 Subject: [PATCH 60/79] redis-cli ASK redirect test: Add retry loop to fix timing issue (#9315) (cherry picked from commit 1c59567a7fe207997eef6197eefa7d508d7fbf9f) --- tests/integration/redis-cli.tcl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/redis-cli.tcl b/tests/integration/redis-cli.tcl index 3ac36c2dd..3117cb6a5 100644 --- a/tests/integration/redis-cli.tcl +++ b/tests/integration/redis-cli.tcl @@ -223,8 +223,13 @@ if {!$::tls} { ;# fake_redis_node doesn't support TLS set p2 [exec $tclsh $script $port2 \ "ASKING" "+OK" \ "SET foo bar" "+OK" &] - # Sleep to make sure both fake nodes have started listening - after 100 + # Make sure both fake nodes have started listening + wait_for_condition 50 50 { + [catch {close [socket "127.0.0.1" $port1]}] == 0 && \ + [catch {close [socket "127.0.0.1" $port2]}] == 0 + } else { + fail "Failed to start fake Redis nodes" + } # Run the cli assert_equal "OK" [run_cli_host_port_db "127.0.0.1" $port1 0 -c SET foo bar] } @@ -238,7 +243,6 @@ if {!$::tls} { ;# fake_redis_node doesn't support TLS test_nontty_cli "No accidental unquoting of input arguments" { run_cli --quoted-input set {"\x41\x41"} quoted-val run_cli set {"\x41\x41"} unquoted-val - assert_equal "quoted-val" [r get AA] assert_equal "unquoted-val" [r get {"\x41\x41"}] } From 44b3a6df77873a981a46c2a07e6ea7ce5c48c235 Mon Sep 17 00:00:00 2001 From: menwen Date: Thu, 5 Aug 2021 16:09:24 +0800 Subject: [PATCH 61/79] Add latency monitor sample when key is deleted via lazy expire (#9317) Fix that there is no sample latency after the key expires via expireIfNeeded(). Some refactoring for shared code. (cherry picked from commit ca559819f7dcd97ba9ef667bf38360a9527d62f6) --- src/db.c | 28 ++++++++++++++++++---------- src/expire.c | 15 +-------------- src/server.h | 1 + 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/db.c b/src/db.c index 840e95e21..05c795114 100644 --- a/src/db.c +++ b/src/db.c @@ -30,6 +30,7 @@ #include "server.h" #include "cluster.h" #include "atomicvar.h" +#include "latency.h" #include #include @@ -1437,6 +1438,22 @@ long long getExpire(redisDb *db, robj *key) { return dictGetSignedIntegerVal(de); } +/* Delete the specified expired key and propagate expire. */ +void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj) { + mstime_t expire_latency; + latencyStartMonitor(expire_latency); + if (server.lazyfree_lazy_expire) + dbAsyncDelete(db,keyobj); + else + dbSyncDelete(db,keyobj); + latencyEndMonitor(expire_latency); + latencyAddSampleIfNeeded("expire-del",expire_latency); + notifyKeyspaceEvent(NOTIFY_EXPIRED,"expired",keyobj,db->id); + signalModifiedKey(NULL, db, keyobj); + propagateExpire(db,keyobj,server.lazyfree_lazy_expire); + server.stat_expiredkeys++; +} + /* Propagate expires into slaves and the AOF file. * When a key expires in the master, a DEL operation for this key is sent * to all the slaves and the AOF file if enabled. @@ -1541,16 +1558,7 @@ int expireIfNeeded(redisDb *db, robj *key) { if (checkClientPauseTimeoutAndReturnIfPaused()) return 1; /* Delete the key */ - if (server.lazyfree_lazy_expire) { - dbAsyncDelete(db,key); - } else { - dbSyncDelete(db,key); - } - server.stat_expiredkeys++; - propagateExpire(db,key,server.lazyfree_lazy_expire); - notifyKeyspaceEvent(NOTIFY_EXPIRED, - "expired",key,db->id); - signalModifiedKey(NULL,db,key); + deleteExpiredKeyAndPropagate(db,key); return 1; } diff --git a/src/expire.c b/src/expire.c index 982301542..f23a79b82 100644 --- a/src/expire.c +++ b/src/expire.c @@ -53,24 +53,11 @@ * to the function to avoid too many gettimeofday() syscalls. */ int activeExpireCycleTryExpire(redisDb *db, dictEntry *de, long long now) { long long t = dictGetSignedIntegerVal(de); - mstime_t expire_latency; if (now > t) { sds key = dictGetKey(de); robj *keyobj = createStringObject(key,sdslen(key)); - - propagateExpire(db,keyobj,server.lazyfree_lazy_expire); - latencyStartMonitor(expire_latency); - if (server.lazyfree_lazy_expire) - dbAsyncDelete(db,keyobj); - else - dbSyncDelete(db,keyobj); - latencyEndMonitor(expire_latency); - latencyAddSampleIfNeeded("expire-del",expire_latency); - notifyKeyspaceEvent(NOTIFY_EXPIRED, - "expired",keyobj,db->id); - signalModifiedKey(NULL, db, keyobj); + deleteExpiredKeyAndPropagate(db,keyobj); decrRefCount(keyobj); - server.stat_expiredkeys++; return 1; } else { return 0; diff --git a/src/server.h b/src/server.h index 07b34c743..abdbca8e5 100644 --- a/src/server.h +++ b/src/server.h @@ -2321,6 +2321,7 @@ void initConfigValues(); /* db.c -- Keyspace access API */ int removeExpire(redisDb *db, robj *key); +void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj); void propagateExpire(redisDb *db, robj *key, int lazy); int keyIsExpired(redisDb *db, robj *key); int expireIfNeeded(redisDb *db, robj *key); From a85fb0283ca385659e319fc168b9c54736ef7eb9 Mon Sep 17 00:00:00 2001 From: sundb Date: Fri, 6 Aug 2021 03:42:20 +0800 Subject: [PATCH 62/79] Sanitize dump payload: fix empty keys when RDB loading and restore command (#9297) When we load rdb or restore command, if we encounter a length of 0, it will result in the creation of an empty key. This could either be a corrupt payload, or a result of a bug (see #8453 ) This PR mainly fixes the following: 1) When restore command will return `Bad data format` error. 2) When loading RDB, we will silently discard the key. Co-authored-by: Oran Agra (cherry picked from commit 8ea777a6a02cae22aeff95f054d810f30b7b69ad) --- src/cluster.c | 2 +- src/rdb.c | 69 +++++++++++++++++++++++++--- src/rdb.h | 7 ++- src/redis-check-rdb.c | 2 +- tests/assets/corrupt_empty_keys.rdb | Bin 0 -> 187 bytes tests/integration/corrupt-dump.tcl | 65 ++++++++++++++++++++++---- 6 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 tests/assets/corrupt_empty_keys.rdb diff --git a/src/cluster.c b/src/cluster.c index 246eb4050..b305353df 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -5177,7 +5177,7 @@ void restoreCommand(client *c) { rioInitWithBuffer(&payload,c->argv[3]->ptr); if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload,key->ptr)) == NULL)) + ((obj = rdbLoadObject(type,&payload,key->ptr,NULL)) == NULL)) { addReplyError(c,"Bad data format"); return; diff --git a/src/rdb.c b/src/rdb.c index 5456c1d80..0fd19a944 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1516,12 +1516,17 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) { } /* Load a Redis object of the specified type from the specified file. - * On success a newly allocated object is returned, otherwise NULL. */ -robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { + * On success a newly allocated object is returned, otherwise NULL. + * When the function returns NULL and if 'error' is not NULL, the + * integer pointed by 'error' is set to the type of error that occurred */ +robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { robj *o = NULL, *ele, *dec; uint64_t len; unsigned int i; + /* Set default error of load object, it will be set to 0 on success. */ + if (error) *error = RDB_LOAD_ERR_OTHER; + int deep_integrity_validation = server.sanitize_dump_payload == SANITIZE_DUMP_YES; if (server.sanitize_dump_payload == SANITIZE_DUMP_CLIENTS) { /* Skip sanitization when loading (an RDB), or getting a RESTORE command @@ -1540,6 +1545,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } else if (rdbtype == RDB_TYPE_LIST) { /* Read list value */ if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; + if (len == 0) goto emptykey; o = createQuicklistObject(); quicklistSetOptions(o->ptr, server.list_max_ziplist_size, @@ -1560,6 +1566,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } else if (rdbtype == RDB_TYPE_SET) { /* Read Set value */ if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; + if (len == 0) goto emptykey; /* Use a regular set when there are too many entries. */ size_t max_entries = server.set_max_intset_entries; @@ -1629,6 +1636,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { zset *zs; if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; + if (zsetlen == 0) goto emptykey; + o = createZsetObject(); zs = o->ptr; @@ -1691,6 +1700,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { len = rdbLoadLen(rdb, NULL); if (len == RDB_LENERR) return NULL; + if (len == 0) goto emptykey; o = createHashObject(); @@ -1807,6 +1817,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { serverAssert(len == 0); } else if (rdbtype == RDB_TYPE_LIST_QUICKLIST) { if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL; + if (len == 0) goto emptykey; + o = createQuicklistObject(); quicklistSetOptions(o->ptr, server.list_max_ziplist_size, server.list_compress_depth); @@ -1940,6 +1952,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } o->type = OBJ_ZSET; o->encoding = OBJ_ENCODING_ZIPLIST; + if (zsetLength(o) == 0) { + zfree(encoded); + o->ptr = NULL; + decrRefCount(o); + goto emptykey; + } + if (zsetLength(o) > server.zset_max_ziplist_entries) zsetConvert(o,OBJ_ENCODING_SKIPLIST); break; @@ -1954,6 +1973,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { } o->type = OBJ_HASH; o->encoding = OBJ_ENCODING_ZIPLIST; + if (hashTypeLength(o) == 0) { + zfree(encoded); + o->ptr = NULL; + decrRefCount(o); + goto emptykey; + } + if (hashTypeLength(o) > server.hash_max_ziplist_entries) hashTypeConvert(o, OBJ_ENCODING_HT); break; @@ -2250,7 +2276,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key) { rdbReportReadError("Unknown RDB encoding type %d",rdbtype); return NULL; } + if (error) *error = 0; return o; + +emptykey: + if (error) *error = RDB_LOAD_ERR_EMPTY_KEY; + return NULL; } /* Mark that we are loading in the global state and setup the fields @@ -2351,6 +2382,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { int type, rdbver; redisDb *db = server.db+0; char buf[1024]; + int error; + long long empty_keys_skipped = 0, expired_keys_skipped = 0, keys_loaded = 0; rdb->update_cksum = rdbLoadProgressCallback; rdb->max_processing_chunk = server.loading_process_events_interval_bytes; @@ -2552,10 +2585,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { if ((key = rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL) goto eoferr; /* Read value */ - if ((val = rdbLoadObject(type,rdb,key)) == NULL) { - sdsfree(key); - goto eoferr; - } + val = rdbLoadObject(type,rdb,key,&error); /* Check if the key already expired. This function is used when loading * an RDB file from disk, either at startup, or when an RDB was @@ -2565,18 +2595,33 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { * Similarly if the RDB is the preamble of an AOF file, we want to * load all the keys as they are, since the log of operations later * assume to work in an exact keyspace state. */ - if (iAmMaster() && + if (val == NULL) { + /* Since we used to have bug that could lead to empty keys + * (See #8453), we rather not fail when empty key is encountered + * in an RDB file, instead we will silently discard it and + * continue loading. */ + if (error == RDB_LOAD_ERR_EMPTY_KEY) { + if(empty_keys_skipped++ < 10) + serverLog(LL_WARNING, "rdbLoadObject skipping empty key: %s", key); + sdsfree(key); + } else { + sdsfree(key); + goto eoferr; + } + } else if (iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now) { sdsfree(key); decrRefCount(val); + expired_keys_skipped++; } else { robj keyobj; initStaticStringObject(keyobj,key); /* Add the new object in the hash table */ int added = dbAddRDBLoad(db,key,val); + keys_loaded++; if (!added) { if (rdbflags & RDBFLAGS_ALLOW_DUP) { /* This flag is useful for DEBUG RELOAD special modes. @@ -2633,6 +2678,16 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) { } } } + + if (empty_keys_skipped) { + serverLog(LL_WARNING, + "Done loading RDB, keys loaded: %lld, keys expired: %lld, empty keys skipped: %lld.", + keys_loaded, expired_keys_skipped, empty_keys_skipped); + } else { + serverLog(LL_WARNING, + "Done loading RDB, keys loaded: %lld, keys expired: %lld.", + keys_loaded, expired_keys_skipped); + } return C_OK; /* Unexpected end of file is handled here calling rdbReportReadError(): diff --git a/src/rdb.h b/src/rdb.h index 00ed5297c..ec34a2082 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -127,6 +127,11 @@ #define RDBFLAGS_REPLICATION (1<<1) /* Load/save for SYNC. */ #define RDBFLAGS_ALLOW_DUP (1<<2) /* Allow duplicated keys when loading.*/ +/* When rdbLoadObject() returns NULL, the err flag is + * set to hold the type of error that occurred */ +#define RDB_LOAD_ERR_EMPTY_KEY 1 /* Error of empty key */ +#define RDB_LOAD_ERR_OTHER 2 /* Any other errors */ + int rdbSaveType(rio *rdb, unsigned char type); int rdbLoadType(rio *rdb); int rdbSaveTime(rio *rdb, time_t t); @@ -145,7 +150,7 @@ void rdbRemoveTempFile(pid_t childpid, int from_signal); int rdbSave(char *filename, rdbSaveInfo *rsi); ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key); size_t rdbSavedObjectLen(robj *o, robj *key); -robj *rdbLoadObject(int type, rio *rdb, sds key); +robj *rdbLoadObject(int type, rio *rdb, sds key, int *error); void backgroundSaveDoneHandler(int exitcode, int bysignal); int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime); ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt); diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 6ddfda7ff..bb655248e 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -308,7 +308,7 @@ int redis_check_rdb(char *rdbfilename, FILE *fp) { rdbstate.keys++; /* Read value */ rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE; - if ((val = rdbLoadObject(type,&rdb,key->ptr)) == NULL) goto eoferr; + if ((val = rdbLoadObject(type,&rdb,key->ptr,NULL)) == NULL) goto eoferr; /* Check if the key already expired. */ if (expiretime != -1 && expiretime < now) rdbstate.already_expired++; diff --git a/tests/assets/corrupt_empty_keys.rdb b/tests/assets/corrupt_empty_keys.rdb new file mode 100644 index 0000000000000000000000000000000000000000..bc2b4f202b75fd17b02b73a879c91e0656a8443c GIT binary patch literal 187 zcmWG?b@2=~Ffg$E#aWb^l3A=fx&=k4iMdHRsRtPTG5ls@U}7##En(o}EG*4T&d$j! zE@9x|sR9bcS7jD}xZK E06t_rssI20 literal 0 HcmV?d00001 diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index fe2537b03..723ce1d64 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -148,6 +148,23 @@ test {corrupt payload: load corrupted rdb with no CRC - #3505} { kill_server $srv ;# let valgrind look for issues } +test {corrupt payload: load corrupted rdb with empty keys} { + set server_path [tmpdir "server.rdb-corruption-empty-keys-test"] + exec cp tests/assets/corrupt_empty_keys.rdb $server_path + start_server [list overrides [list "dir" $server_path "dbfilename" "corrupt_empty_keys.rdb"]] { + r select 0 + assert_equal [r dbsize] 0 + + verify_log_message 0 "*skipping empty key: set*" 0 + verify_log_message 0 "*skipping empty key: quicklist*" 0 + verify_log_message 0 "*skipping empty key: hash*" 0 + verify_log_message 0 "*skipping empty key: hash_ziplist*" 0 + verify_log_message 0 "*skipping empty key: zset*" 0 + verify_log_message 0 "*skipping empty key: zset_ziplist*" 0 + verify_log_message 0 "*empty keys skipped: 6*" 0 + } +} + test {corrupt payload: listpack invalid size header} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no @@ -328,12 +345,13 @@ test {corrupt payload: fuzzer findings - leak in rdbloading due to dup entry in } } -test {corrupt payload: fuzzer findings - empty intset div by zero} { +test {corrupt payload: fuzzer findings - empty intset} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload no r debug set-skip-checksum-validation 1 - r RESTORE _setbig 0 "\x02\xC0\xC0\x06\x02\x5F\x39\xC0\x02\x02\x5F\x33\xC0\x00\x02\x5F\x31\xC0\x04\xC0\x08\x02\x5F\x37\x02\x5F\x35\x09\x00\xC5\xD4\x6D\xBA\xAD\x14\xB7\xE7" - catch {r SRANDMEMBER _setbig } + catch {r RESTORE _setbig 0 "\x02\xC0\xC0\x06\x02\x5F\x39\xC0\x02\x02\x5F\x33\xC0\x00\x02\x5F\x31\xC0\x04\xC0\x08\x02\x5F\x37\x02\x5F\x35\x09\x00\xC5\xD4\x6D\xBA\xAD\x14\xB7\xE7"} err + assert_match "*Bad data format*" $err + r ping } } @@ -507,14 +525,13 @@ test {corrupt payload: fuzzer findings - valgrind invalid read} { } } -test {corrupt payload: fuzzer findings - HRANDFIELD on bad ziplist} { +test {corrupt payload: fuzzer findings - empty hash ziplist} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload yes r debug set-skip-checksum-validation 1 - r RESTORE _int 0 "\x04\xC0\x01\x09\x00\xF6\x8A\xB6\x7A\x85\x87\x72\x4D" - catch {r HRANDFIELD _int} - assert_equal [count_log_message 0 "crashed by signal"] 0 - assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + catch {r RESTORE _int 0 "\x04\xC0\x01\x09\x00\xF6\x8A\xB6\x7A\x85\x87\x72\x4D"} err + assert_match "*Bad data format*" $err + r ping } } @@ -529,5 +546,37 @@ test {corrupt payload: fuzzer findings - stream with no records} { } } +test {corrupt payload: fuzzer findings - empty quicklist} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { + r restore key 0 "\x0E\xC0\x2B\x15\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\xE0\x62\x58\xEA\xDF\x22\x00\x00\x00\xFF\x09\x00\xDF\x35\xD2\x67\xDC\x0E\x89\xAB" replace + } err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - empty zset} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore key 0 "\x05\xC0\x01\x09\x00\xF6\x8A\xB6\x7A\x85\x87\x72\x4D"} err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - hash with len of 0} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore key 0 "\x04\xC0\x21\x09\x00\xF6\x8A\xB6\x7A\x85\x87\x72\x4D"} err + assert_match "*Bad data format*" $err + r ping + } +} + } ;# tags From 1334180f8289fa66b1afb524e3512f2f1eafcb2f Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Aug 2021 22:56:14 +0300 Subject: [PATCH 63/79] Improvements to corrupt payload sanitization (#9321) Recently we found two issues in the fuzzer tester: #9302 #9285 After fixing them, more problems surfaced and this PR (as well as #9297) aims to fix them. Here's a list of the fixes - Prevent an overflow when allocating a dict hashtable - Prevent OOM when attempting to allocate a huge string - Prevent a few invalid accesses in listpack - Improve sanitization of listpack first entry - Validate integrity of stream consumer groups PEL - Validate integrity of stream listpack entry IDs - Validate ziplist tail followed by extra data which start with 0xff Co-authored-by: sundb (cherry picked from commit 0c90370e6d71cc68e4d9cc79a0d8b1e768712a5b) --- src/dict.c | 4 ++ src/listpack.c | 33 +++++++++--- src/listpack.h | 1 + src/object.c | 15 ++++++ src/rdb.c | 25 ++++++++- src/server.h | 2 + src/t_stream.c | 6 ++- src/ziplist.c | 4 ++ tests/integration/corrupt-dump.tcl | 86 ++++++++++++++++++++++++++++++ 9 files changed, 166 insertions(+), 10 deletions(-) diff --git a/src/dict.c b/src/dict.c index 21c616e6f..1bbdc211e 100644 --- a/src/dict.c +++ b/src/dict.c @@ -154,6 +154,10 @@ int _dictExpand(dict *d, unsigned long size, int* malloc_failed) dictht n; /* the new hash table */ unsigned long realsize = _dictNextPower(size); + /* Detect overflows */ + if (realsize < size || realsize * sizeof(dictEntry*) < realsize) + return DICT_ERR; + /* Rehashing to the same table size is not useful. */ if (realsize == d->ht[0].size) return DICT_ERR; diff --git a/src/listpack.c b/src/listpack.c index 27622d4a5..8424da87f 100644 --- a/src/listpack.c +++ b/src/listpack.c @@ -131,6 +131,8 @@ assert((p) >= (lp)+LP_HDR_SIZE && (p)+(len) < (lp)+lpGetTotalBytes((lp))); \ } while (0) +static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p); + /* Convert a string into a signed 64 bit integer. * The function returns 1 if the string could be parsed into a (non-overflowing) * signed 64 bit int, 0 otherwise. The 'value' will be set to the parsed value @@ -453,8 +455,8 @@ unsigned char *lpSkip(unsigned char *p) { unsigned char *lpNext(unsigned char *lp, unsigned char *p) { assert(p); p = lpSkip(p); - ASSERT_INTEGRITY(lp, p); if (p[0] == LP_EOF) return NULL; + lpAssertValidEntry(lp, lpBytes(lp), p); return p; } @@ -468,16 +470,17 @@ unsigned char *lpPrev(unsigned char *lp, unsigned char *p) { uint64_t prevlen = lpDecodeBacklen(p); prevlen += lpEncodeBacklen(NULL,prevlen); p -= prevlen-1; /* Seek the first byte of the previous entry. */ - ASSERT_INTEGRITY(lp, p); + lpAssertValidEntry(lp, lpBytes(lp), p); return p; } /* Return a pointer to the first element of the listpack, or NULL if the * listpack has no elements. */ unsigned char *lpFirst(unsigned char *lp) { - lp += LP_HDR_SIZE; /* Skip the header. */ - if (lp[0] == LP_EOF) return NULL; - return lp; + unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */ + if (p[0] == LP_EOF) return NULL; + lpAssertValidEntry(lp, lpBytes(lp), p); + return p; } /* Return a pointer to the last element of the listpack, or NULL if the @@ -861,6 +864,13 @@ unsigned char *lpSeek(unsigned char *lp, long index) { } } +/* Same as lpFirst but without validation assert, to be used right before lpValidateNext. */ +unsigned char *lpValidateFirst(unsigned char *lp) { + unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */ + if (p[0] == LP_EOF) return NULL; + return p; +} + /* Validate the integrity of a single listpack entry and move to the next one. * The input argument 'pp' is a reference to the current record and is advanced on exit. * Returns 1 if valid, 0 if invalid. */ @@ -872,6 +882,10 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { if (!p) return 0; + /* Before accessing p, make sure it's valid. */ + if (OUT_OF_RANGE(p)) + return 0; + if (*p == LP_EOF) { *pp = NULL; return 1; @@ -908,6 +922,11 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) { #undef OUT_OF_RANGE } +/* Validate that the entry doesn't reach outside the listpack allocation. */ +static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p) { + assert(lpValidateNext(lp, &p, lpbytes)); +} + /* Validate the integrity of the data structure. * when `deep` is 0, only the integrity of the header is validated. * when `deep` is 1, we scan all the entries one by one. */ @@ -930,8 +949,8 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){ /* Validate the invividual entries. */ uint32_t count = 0; - unsigned char *p = lpFirst(lp); - while(p) { + unsigned char *p = lp + LP_HDR_SIZE; + while(p && p[0] != LP_EOF) { if (!lpValidateNext(lp, &p, bytes)) return 0; count++; diff --git a/src/listpack.h b/src/listpack.h index f87622c18..9a5115bad 100644 --- a/src/listpack.h +++ b/src/listpack.h @@ -60,6 +60,7 @@ unsigned char *lpPrev(unsigned char *lp, unsigned char *p); uint32_t lpBytes(unsigned char *lp); unsigned char *lpSeek(unsigned char *lp, long index); int lpValidateIntegrity(unsigned char *lp, size_t size, int deep); +unsigned char *lpValidateFirst(unsigned char *lp); int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes); #endif diff --git a/src/object.c b/src/object.c index c6381a231..23920fb2c 100644 --- a/src/object.c +++ b/src/object.c @@ -123,6 +123,21 @@ robj *createStringObject(const char *ptr, size_t len) { return createRawStringObject(ptr,len); } +/* Same as CreateRawStringObject, can return NULL if allocation fails */ +robj *tryCreateRawStringObject(const char *ptr, size_t len) { + sds str = sdstrynewlen(ptr,len); + if (!str) return NULL; + return createObject(OBJ_STRING, str); +} + +/* Same as createStringObject, can return NULL if allocation fails */ +robj *tryCreateStringObject(const char *ptr, size_t len) { + if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT) + return createEmbeddedStringObject(ptr,len); + else + return tryCreateRawStringObject(ptr,len); +} + /* Create a string object from a long long value. When possible returns a * shared integer object, or at least an integer encoded one. * diff --git a/src/rdb.c b/src/rdb.c index 0fd19a944..051a0d87e 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -545,8 +545,12 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) { } return buf; } else { - robj *o = encode ? createStringObject(SDS_NOINIT,len) : - createRawStringObject(SDS_NOINIT,len); + robj *o = encode ? tryCreateStringObject(SDS_NOINIT,len) : + tryCreateRawStringObject(SDS_NOINIT,len); + if (!o) { + serverLog(server.loading? LL_WARNING: LL_VERBOSE, "rdbGenericLoadStringObject failed allocating %llu bytes", len); + return NULL; + } if (len && rioRead(rdb,o->ptr,len) == 0) { decrRefCount(o); return NULL; @@ -2210,6 +2214,23 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { } } } + + /* Verify that each PEL eventually got a consumer assigned to it. */ + if (deep_integrity_validation) { + raxIterator ri_cg_pel; + raxStart(&ri_cg_pel,cgroup->pel); + raxSeek(&ri_cg_pel,"^",NULL,0); + while(raxNext(&ri_cg_pel)) { + streamNACK *nack = ri_cg_pel.data; + if (!nack->consumer) { + raxStop(&ri_cg_pel); + rdbReportCorruptRDB("Stream CG PEL entry without consumer"); + decrRefCount(o); + return NULL; + } + } + raxStop(&ri_cg_pel); + } } } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) { uint64_t moduleid = rdbLoadLen(rdb,NULL); diff --git a/src/server.h b/src/server.h index abdbca8e5..74bd5447a 100644 --- a/src/server.h +++ b/src/server.h @@ -1973,6 +1973,8 @@ robj *createObject(int type, void *ptr); robj *createStringObject(const char *ptr, size_t len); robj *createRawStringObject(const char *ptr, size_t len); robj *createEmbeddedStringObject(const char *ptr, size_t len); +robj *tryCreateRawStringObject(const char *ptr, size_t len); +robj *tryCreateStringObject(const char *ptr, size_t len); robj *dupStringObject(const robj *o); int isSdsRepresentableAsLongLong(sds s, long long *llval); int isObjectRepresentableAsLongLong(robj *o, long long *llongval); diff --git a/src/t_stream.c b/src/t_stream.c index 574195ee3..7e67c1b0e 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3590,7 +3590,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) { /* In non-deep mode we just validated the listpack header (encoded size) */ if (!deep) return 1; - next = p = lpFirst(lp); + next = p = lpValidateFirst(lp); if (!lpValidateNext(lp, &next, size)) return 0; if (!p) return 0; @@ -3629,7 +3629,11 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) { /* entry id */ p = next; if (!lpValidateNext(lp, &next, size)) return 0; + lpGetIntegerIfValid(p, &valid_record); + if (!valid_record) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; + lpGetIntegerIfValid(p, &valid_record); + if (!valid_record) return 0; if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) { /* num-of-fields */ diff --git a/src/ziplist.c b/src/ziplist.c index fdc1bb9e1..74c033ce2 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1537,6 +1537,10 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep, count++; } + /* Make sure 'p' really does point to the end of the ziplist. */ + if (p != zl + bytes - ZIPLIST_END_SIZE) + return 0; + /* Make sure the entry really do point to the start of the last entry. */ if (prev != ZIPLIST_ENTRY_TAIL(zl)) return 0; diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 723ce1d64..896571c2f 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -546,6 +546,92 @@ test {corrupt payload: fuzzer findings - stream with no records} { } } +test {corrupt payload: fuzzer findings - quicklist ziplist tail followed by extra data which start with 0xff} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch { + r restore key 0 "\x0E\x01\x11\x11\x00\x00\x00\x0A\x00\x00\x00\x01\x00\x00\xF6\xFF\xB0\x6C\x9C\xFF\x09\x00\x9C\x37\x47\x49\x4D\xDE\x94\xF5" replace + } err + assert_match "*Bad data format*" $err + verify_log_message 0 "*integrity check failed*" 0 + } +} + +test {corrupt payload: fuzzer findings - dict init to huge size} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + catch {r restore key 0 "\x02\x81\xC0\x00\x02\x5F\x31\xC0\x02\x09\x00\xB2\x1B\xE5\x17\x2E\x15\xF4\x6C" replace} err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - huge string} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore key 0 "\x00\x81\x01\x09\x00\xF6\x2B\xB6\x7A\x85\x87\x72\x4D"} err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - stream PEL without consumer} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x3B\x40\x42\x19\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x20\x10\x00\x00\x20\x01\x00\x01\x20\x03\x02\x05\x01\x03\x20\x05\x40\x00\x04\x82\x5F\x31\x03\x05\x60\x19\x80\x32\x02\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x02\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x08\xF0\xB2\x34\x01\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x01\x13\x41\x6C\x69\x63\x65\x35\xB2\xF0\x08\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x08\xF0\xB2\x34\x00\x00\x00\x00\x00\x00\x00\x01\x09\x00\x28\x2F\xE0\xC5\x04\xBB\xA7\x31"} err + assert_match "*Bad data format*" $err + #catch {r XINFO STREAM _stream FULL } + r ping + } +} + +test {corrupt payload: fuzzer findings - stream listpack valgrind issue} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x09\x5E\x94\xFF\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x03\x01\x25\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x32\x01\x00\x01\x01\x01\x02\x01\xF0\x01\xFF\x02\x81\x00\x00\x01\x7B\x09\x5E\x95\x31\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x09\x5E\x95\x24\x00\x01\x00\x00\x01\x7B\x09\x5E\x95\x24\x00\x00\x00\x00\x00\x00\x00\x00\x5C\x95\x5E\x09\x7B\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x4B\x95\x5E\x09\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x09\x5E\x95\x24\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\x19\x29\x94\xDF\x76\xF8\x1A\xC6" + catch {r XINFO STREAM _stream FULL } + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - stream with bad lpFirst} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x0E\x52\xD2\xEC\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x02\xF7\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x05\x01\x03\x01\x01\x01\x00\x01\x01\x01\x82\x5F\x31\x03\x05\x01\x02\x01\x01\x01\x01\x01\x01\x01\x02\x01\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x0E\x52\xD2\xED\x01\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x0E\x52\xD2\xED\x00\x01\x00\x00\x01\x7B\x0E\x52\xD2\xED\x00\x00\x00\x00\x00\x00\x00\x00\xED\xD2\x52\x0E\x7B\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\xED\xD2\x52\x0E\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x0E\x52\xD2\xED\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xAC\x05\xC9\x97\x5D\x45\x80\xB3"} err + assert_match "*Bad data format*" $err + r ping + } +} + +test {corrupt payload: fuzzer findings - stream listpack lpPrev valgrind issue} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload no + r debug set-skip-checksum-validation 1 + r restore _stream 0 "\x0F\x01\x10\x00\x00\x01\x7B\x0E\xAE\x66\x36\x00\x00\x00\x00\x00\x00\x00\x00\x40\x42\x42\x00\x00\x00\x18\x00\x02\x01\x01\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x00\x01\x02\x01\x00\x01\x00\x01\x01\x01\x00\x01\x1D\x01\x03\x01\x24\x01\x00\x01\x01\x69\x82\x5F\x31\x03\x05\x01\x02\x01\x33\x01\x00\x01\x01\x01\x02\x01\x05\x01\xFF\x02\x81\x00\x00\x01\x7B\x0E\xAE\x66\x69\x00\x01\x07\x6D\x79\x67\x72\x6F\x75\x70\x81\x00\x00\x01\x7B\x0E\xAE\x66\x5A\x00\x01\x00\x00\x01\x7B\x0E\xAE\x66\x5A\x00\x00\x00\x00\x00\x00\x00\x00\x94\x66\xAE\x0E\x7B\x01\x00\x00\x01\x01\x05\x41\x6C\x69\x63\x65\x83\x66\xAE\x0E\x7B\x01\x00\x00\x01\x00\x00\x01\x7B\x0E\xAE\x66\x5A\x00\x00\x00\x00\x00\x00\x00\x00\x09\x00\xD5\xD7\xA5\x5C\x63\x1C\x09\x40" + catch {r XREVRANGE _stream 1618622681 606195012389} + assert_equal [count_log_message 0 "crashed by signal"] 0 + assert_equal [count_log_message 0 "ASSERTION FAILED"] 1 + } +} + +test {corrupt payload: fuzzer findings - stream with non-integer entry id} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r config set sanitize-dump-payload yes + r debug set-skip-checksum-validation 1 + catch {r restore _streambig 0 "\x0F\x03\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x80\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB2\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x56\x40\x61\x18\x61\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x40\x0B\x03\x01\x01\xDF\xFB\x20\x05\x02\x82\x5F\x37\x60\x1A\x20\x0E\x00\xFC\x20\x05\x00\x08\xC0\x1B\x00\xFD\x20\x0C\x02\x82\x5F\x39\x20\x1B\x00\xFF\x10\x00\x00\x01\x7B\x13\x34\xC3\xB3\x00\x00\x00\x00\x00\x00\x00\x03\xC3\x3D\x40\x4A\x18\x4A\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x60\x07\x02\xDF\xFD\x02\xC0\x23\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2D\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x7B\x13\x34\xC3\xB4\x00\x00\x09\x00\x9D\xBD\xD5\xB9\x33\xC4\xC5\xFF"} err + #catch {r XINFO STREAM _streambig FULL } + assert_match "*Bad data format*" $err + r ping + } +} + test {corrupt payload: fuzzer findings - empty quicklist} { start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { r config set sanitize-dump-payload yes From 94bf9b175e96f9459f42f410543810b8915786dc Mon Sep 17 00:00:00 2001 From: DarrenJiang13 Date: Sat, 7 Aug 2021 10:27:24 +0800 Subject: [PATCH 64/79] [BUGFIX] Add some missed error statistics (#9328) add error counting for some missed behaviors. (cherry picked from commit 43eb0ce3bf76a5d287b93a767bead9ad6230a1ad) --- src/db.c | 2 +- src/multi.c | 10 +++++++--- src/object.c | 3 +-- src/server.h | 1 + tests/unit/info.tcl | 7 ++++--- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/db.c b/src/db.c index 05c795114..6f6d696e1 100644 --- a/src/db.c +++ b/src/db.c @@ -165,7 +165,7 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) { robj *lookupKeyWrite(redisDb *db, robj *key) { return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE); } -static void SentReplyOnKeyMiss(client *c, robj *reply){ +void SentReplyOnKeyMiss(client *c, robj *reply){ serverAssert(sdsEncodedObject(reply)); sds rep = reply->ptr; if (sdslen(rep) > 1 && rep[0] == '-'){ diff --git a/src/multi.c b/src/multi.c index 3a157bee6..ebbdabbf0 100644 --- a/src/multi.c +++ b/src/multi.c @@ -179,9 +179,13 @@ void execCommand(client *c) { * A failed EXEC in the first case returns a multi bulk nil object * (technically it is not an error but a special behavior), while * in the second an EXECABORT error is returned. */ - if (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC)) { - addReply(c, c->flags & CLIENT_DIRTY_EXEC ? shared.execaborterr : - shared.nullarray[c->resp]); + if (c->flags & (CLIENT_DIRTY_CAS | CLIENT_DIRTY_EXEC)) { + if (c->flags & CLIENT_DIRTY_EXEC) { + addReplyErrorObject(c, shared.execaborterr); + } else { + addReply(c, shared.nullarray[c->resp]); + } + discardTransaction(c); return; } diff --git a/src/object.c b/src/object.c index 23920fb2c..8a5e80a6f 100644 --- a/src/object.c +++ b/src/object.c @@ -1246,8 +1246,7 @@ robj *objectCommandLookup(client *c, robj *key) { robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply) { robj *o = objectCommandLookup(c,key); - - if (!o) addReply(c, reply); + if (!o) SentReplyOnKeyMiss(c, reply); return o; } diff --git a/src/server.h b/src/server.h index 74bd5447a..4edb7d3a0 100644 --- a/src/server.h +++ b/src/server.h @@ -2339,6 +2339,7 @@ robj *lookupKeyReadWithFlags(redisDb *db, robj *key, int flags); robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags); robj *objectCommandLookup(client *c, robj *key); robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply); +void SentReplyOnKeyMiss(client *c, robj *reply); int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle, long long lru_clock, int lru_multiplier); #define LOOKUP_NONE 0 diff --git a/tests/unit/info.tcl b/tests/unit/info.tcl index 0602e7147..61be4a0d1 100644 --- a/tests/unit/info.tcl +++ b/tests/unit/info.tcl @@ -110,11 +110,12 @@ start_server {tags {"info"}} { catch {r exec} e assert_match {EXECABORT*} $e assert_match {*count=1*} [errorstat ERR] - assert_equal [s total_error_replies] 1 + assert_match {*count=1*} [errorstat EXECABORT] + assert_equal [s total_error_replies] 2 assert_match {*calls=0,*,rejected_calls=1,failed_calls=0} [cmdstat set] assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat multi] - assert_match {*calls=1,*,rejected_calls=0,failed_calls=0} [cmdstat exec] - assert_equal [s total_error_replies] 1 + assert_match {*calls=1,*,rejected_calls=0,failed_calls=1} [cmdstat exec] + assert_equal [s total_error_replies] 2 r config resetstat assert_match {} [errorstat ERR] } From b2f4ca8c64c98a5af7225cbbcb7dcecf16159d80 Mon Sep 17 00:00:00 2001 From: sundb Date: Mon, 9 Aug 2021 22:13:46 +0800 Subject: [PATCH 65/79] Sanitize dump payload: handle remaining empty key when RDB loading and restore command (#9349) This commit mainly fixes empty keys due to RDB loading and restore command, which was omitted in #9297. 1) When loading quicklsit, if all the ziplists in the quicklist are empty, NULL will be returned. If only some of the ziplists are empty, then we will skip the empty ziplists silently. 2) When loading hash zipmap, if zipmap is empty, sanitization check will fail. 3) When loading hash ziplist, if ziplist is empty, NULL will be returned. 4) Add RDB loading test with sanitize. (cherry picked from commit cbda492909cd2fff25263913cd2e1f00bc48a541) --- src/rdb.c | 22 +++++++++++- src/ziplist.c | 2 +- src/zipmap.c | 3 ++ tests/assets/corrupt_empty_keys.rdb | Bin 187 -> 261 bytes tests/integration/corrupt-dump.tcl | 50 ++++++++++++++++++++-------- 5 files changed, 62 insertions(+), 15 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 051a0d87e..3cc84f64f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1842,7 +1842,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { zfree(zl); return NULL; } - quicklistAppendZiplist(o->ptr, zl); + + /* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */ + if (ziplistLen(zl) == 0) { + zfree(zl); + continue; + } else { + quicklistAppendZiplist(o->ptr, zl); + } + } + + if (quicklistCount(o->ptr) == 0) { + decrRefCount(o); + goto emptykey; } } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP || rdbtype == RDB_TYPE_LIST_ZIPLIST || @@ -1927,6 +1939,14 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { decrRefCount(o); return NULL; } + + if (ziplistLen(encoded) == 0) { + zfree(encoded); + o->ptr = NULL; + decrRefCount(o); + goto emptykey; + } + o->type = OBJ_LIST; o->encoding = OBJ_ENCODING_ZIPLIST; listTypeConvert(o,OBJ_ENCODING_QUICKLIST); diff --git a/src/ziplist.c b/src/ziplist.c index 74c033ce2..b3b27ce36 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1542,7 +1542,7 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep, return 0; /* Make sure the entry really do point to the start of the last entry. */ - if (prev != ZIPLIST_ENTRY_TAIL(zl)) + if (prev != NULL && prev != ZIPLIST_ENTRY_TAIL(zl)) return 0; /* Check that the count in the header is correct */ diff --git a/src/zipmap.c b/src/zipmap.c index c24e81355..21a608843 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -430,6 +430,9 @@ int zipmapValidateIntegrity(unsigned char *zm, size_t size, int deep) { return 0; } + /* check that the zipmap is not empty. */ + if (count == 0) return 0; + /* check that the count in the header is correct */ if (zm[0] != ZIPMAP_BIGLEN && zm[0] != count) return 0; diff --git a/tests/assets/corrupt_empty_keys.rdb b/tests/assets/corrupt_empty_keys.rdb index bc2b4f202b75fd17b02b73a879c91e0656a8443c..8f260d49333fbe04e749d0d4ea2f9a7349994cc1 100644 GIT binary patch delta 148 zcmdnZ*vd4)%vfGmF!2{hX>n?bZf;ICWy4880hKRdH$w0~b$DW^u{H2_mk1d>~eQVQFS^Hi*T*Cxa>wpPE}x hQW;;BSpX7e Date: Sun, 15 Aug 2021 04:52:44 +0800 Subject: [PATCH 66/79] Fix the wrong detection of sync_file_range system call (#9371) If we want to check `defined(SYNC_FILE_RANGE_WAIT_BEFORE)`, we should include fcntl.h. otherwise, SYNC_FILE_RANGE_WAIT_BEFORE is not defined, and there is alway not `sync_file_range` system call. Introduced by #8532 (cherry picked from commit 8edc3cd62c0d0508b68c887610ca53b632b8165b) --- src/config.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.h b/src/config.h index 56c1ab6ae..2f351a1b4 100644 --- a/src/config.h +++ b/src/config.h @@ -36,6 +36,7 @@ #ifdef __linux__ #include +#include #endif /* Define redis_fstat to fstat or fstat64() */ From b69f619f17af5c474643e222a63a89d5df1c7bad Mon Sep 17 00:00:00 2001 From: sundb Date: Fri, 20 Aug 2021 15:37:45 +0800 Subject: [PATCH 67/79] Sanitize dump payload: fix double free after insert dup nodekey to stream rax and returns 0 (#9399) (cherry picked from commit 492d8d09613cff88f15dcef98732392b8d509eb1) --- src/rdb.c | 7 ++++--- tests/integration/corrupt-dump.tcl | 10 ++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 3cc84f64f..3a1eeb915 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -2072,7 +2072,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { } /* Insert the key in the radix tree. */ - int retval = raxInsert(s->rax, + int retval = raxTryInsert(s->rax, (unsigned char*)nodekey,sizeof(streamID),lp,NULL); sdsfree(nodekey); if (!retval) { @@ -2161,7 +2161,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { streamFreeNACK(nack); return NULL; } - if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) { + if (!raxTryInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) { rdbReportCorruptRDB("Duplicated global PEL entry " "loading stream consumer group"); decrRefCount(o); @@ -2225,11 +2225,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error) { * loading the global PEL. Then set the same shared * NACK structure also in the consumer-specific PEL. */ nack->consumer = consumer; - if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) { + if (!raxTryInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) { rdbReportCorruptRDB("Duplicated consumer PEL entry " " loading a stream consumer " "group"); decrRefCount(o); + streamFreeNACK(nack); return NULL; } } diff --git a/tests/integration/corrupt-dump.tcl b/tests/integration/corrupt-dump.tcl index 75bdf0f7f..fbdd3c373 100644 --- a/tests/integration/corrupt-dump.tcl +++ b/tests/integration/corrupt-dump.tcl @@ -688,5 +688,15 @@ test {corrupt payload: fuzzer findings - hash with len of 0} { } } +test {corrupt payload: fuzzer findings - stream double free listpack when insert dup node to rax returns 0} { + start_server [list overrides [list loglevel verbose use-exit-on-panic yes crash-memcheck-enabled no] ] { + r debug set-skip-checksum-validation 1 + r config set sanitize-dump-payload yes + catch { r restore _stream 0 "\x0F\x03\x10\x00\x00\x01\x7B\x60\x5A\x23\x79\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x40\x4F\x40\x5C\x18\x5C\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x00\x01\x20\x03\x00\x05\x20\x1C\x40\x09\x05\x01\x01\x82\x5F\x31\x03\x80\x0D\x00\x02\x20\x0D\x00\x02\xA0\x19\x00\x03\x20\x0B\x02\x82\x5F\x33\xA0\x19\x00\x04\x20\x0D\x00\x04\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x60\x5A\x23\x79\x00\x00\x00\x00\x00\x00\x00\x05\xC3\x40\x51\x40\x5E\x18\x5E\x00\x00\x00\x24\x00\x05\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x06\x01\x01\x82\x5F\x35\x03\x05\x20\x1E\x40\x0B\x03\x01\x01\x06\x01\x80\x0B\x00\x02\x20\x0B\x02\x82\x5F\x37\xA0\x19\x00\x03\x20\x0D\x00\x08\xA0\x19\x00\x04\x20\x0B\x02\x82\x5F\x39\x20\x19\x00\xFF\x10\x00\x00\x01\x7B\x60\x5A\x23\x79\x00\x00\x00\x00\x00\x00\x00\x00\xC3\x3B\x40\x49\x18\x49\x00\x00\x00\x15\x00\x02\x01\x00\x01\x02\x01\x84\x69\x74\x65\x6D\x05\x85\x76\x61\x6C\x75\x65\x06\x40\x10\x00\x00\x20\x01\x40\x00\x00\x05\x20\x07\x40\x09\xC0\x22\x09\x01\x01\x86\x75\x6E\x69\x71\x75\x65\x07\xA0\x2C\x02\x08\x01\xFF\x0C\x81\x00\x00\x01\x7B\x60\x5A\x23\x7A\x01\x00\x09\x00\x9C\x8F\x1E\xBF\x2E\x05\x59\x09" replace } err + assert_match "*Bad data format*" $err + r ping + } +} + } ;# tags From 69ffd6cbc5735e059cbf8d2bfdd0affc4b582157 Mon Sep 17 00:00:00 2001 From: sundb Date: Thu, 2 Sep 2021 16:07:51 +0800 Subject: [PATCH 68/79] Fix the timing of read and write events under kqueue (#9416) Normally we execute the read event first and then the write event. When the barrier is set, we will do it reverse. However, under `kqueue`, if an `fd` has both read and write events, reading the event using `kevent` will generate two events, which will result in uncontrolled read and write timing. This also means that the guarantees of AOF `appendfsync` = `always` are not met on MacOS without this fix. The main change to this pr is to cache the events already obtained when reading them, so that if the same `fd` occurs again, only the mask in the cache is updated, rather than a new event is generated. This was exposed by the following test failure on MacOS: ``` *** [err]: AOF fsync always barrier issue in tests/integration/aof.tcl Expected 544 != 544 (context: type eval line 26 cmd {assert {$size1 != $size2}} proc ::test) ``` (cherry picked from commit 306a5ccd2d053ff653988b61a779e3cbce408874) --- src/ae_kqueue.c | 62 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index b146f2519..6d62720b2 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -36,8 +36,29 @@ typedef struct aeApiState { int kqfd; struct kevent *events; + + /* Events mask for merge read and write event. + * To reduce memory consumption, we use 2 bits to store the mask + * of an event, so that 1 byte will store the mask of 4 events. */ + char *eventsMask; } aeApiState; +#define EVENT_MASK_MALLOC_SIZE(sz) (((sz) + 3) / 4) +#define EVENT_MASK_OFFSET(fd) ((fd) % 4 * 2) +#define EVENT_MASK_ENCODE(fd, mask) (((mask) & 0x3) << EVENT_MASK_OFFSET(fd)) + +static inline int getEventMask(const char *eventsMask, int fd) { + return (eventsMask[fd/4] >> EVENT_MASK_OFFSET(fd)) & 0x3; +} + +static inline void addEventMask(char *eventsMask, int fd, int mask) { + eventsMask[fd/4] |= EVENT_MASK_ENCODE(fd, mask); +} + +static inline void resetEventMask(char *eventsMask, int fd) { + eventsMask[fd/4] &= ~EVENT_MASK_ENCODE(fd, 0x3); +} + static int aeApiCreate(aeEventLoop *eventLoop) { aeApiState *state = zmalloc(sizeof(aeApiState)); @@ -54,6 +75,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return -1; } anetCloexec(state->kqfd); + state->eventsMask = zmalloc(EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(eventLoop->setsize)); eventLoop->apidata = state; return 0; } @@ -62,6 +85,8 @@ static int aeApiResize(aeEventLoop *eventLoop, int setsize) { aeApiState *state = eventLoop->apidata; state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + state->eventsMask = zrealloc(state->eventsMask, EVENT_MASK_MALLOC_SIZE(setsize)); + memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(setsize)); return 0; } @@ -70,6 +95,7 @@ static void aeApiFree(aeEventLoop *eventLoop) { close(state->kqfd); zfree(state->events); + zfree(state->eventsMask); zfree(state); } @@ -120,15 +146,37 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { if (retval > 0) { int j; - numevents = retval; - for(j = 0; j < numevents; j++) { - int mask = 0; + /* Normally we execute the read event first and then the write event. + * When the barrier is set, we will do it reverse. + * + * However, under kqueue, read and write events would be separate + * events, which would make it impossible to control the order of + * reads and writes. So we store the event's mask we've got and merge + * the same fd events later. */ + for (j = 0; j < retval; j++) { struct kevent *e = state->events+j; + int fd = e->ident; + int mask = 0; - if (e->filter == EVFILT_READ) mask |= AE_READABLE; - if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; - eventLoop->fired[j].fd = e->ident; - eventLoop->fired[j].mask = mask; + if (e->filter == EVFILT_READ) mask = AE_READABLE; + else if (e->filter == EVFILT_WRITE) mask = AE_WRITABLE; + addEventMask(state->eventsMask, fd, mask); + } + + /* Re-traversal to merge read and write events, and set the fd's mask to + * 0 so that events are not added again when the fd is encountered again. */ + numevents = 0; + for (j = 0; j < retval; j++) { + struct kevent *e = state->events+j; + int fd = e->ident; + int mask = getEventMask(state->eventsMask, fd); + + if (mask) { + eventLoop->fired[numevents].fd = fd; + eventLoop->fired[numevents].mask = mask; + resetEventMask(state->eventsMask, fd); + numevents++; + } } } return numevents; From 7c2c8608a79c94f5bfa3f8f167d09ca7242a8202 Mon Sep 17 00:00:00 2001 From: guybe7 Date: Thu, 2 Sep 2021 16:19:27 +0200 Subject: [PATCH 69/79] Fix two minor bugs (MIGRATE key args and getKeysUsingCommandTable) (#9455) 1. MIGRATE has a potnetial key arg in argv[3]. It should be reflected in the command table. 2. getKeysUsingCommandTable should never free getKeysResult, it is always freed by the caller) The reason we never encountered this double-free bug is that almost always getKeysResult uses the statis buffer and doesn't allocate a new one. (cherry picked from commit 6aa2285e32a6bc16fe2938bfb40d833db7d3752d) --- src/db.c | 1 - src/server.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index 6f6d696e1..c8d6199d6 100644 --- a/src/db.c +++ b/src/db.c @@ -1623,7 +1623,6 @@ int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, get * return no keys and expect the command implementation to report * an arity or syntax error. */ if (cmd->flags & CMD_MODULE || cmd->arity < 0) { - getKeysFreeResult(result); result->numkeys = 0; return 0; } else { diff --git a/src/server.c b/src/server.c index 43cb81b85..065028e27 100644 --- a/src/server.c +++ b/src/server.c @@ -870,7 +870,7 @@ struct redisCommand redisCommandTable[] = { {"migrate",migrateCommand,-6, "write random @keyspace @dangerous", - 0,migrateGetKeys,0,0,0,0,0,0}, + 0,migrateGetKeys,3,3,1,0,0,0}, {"asking",askingCommand,1, "fast @keyspace", From b9c6d6a321a41a86a46ae53774117a314a071468 Mon Sep 17 00:00:00 2001 From: Madelyn Olson <34459052+madolson@users.noreply.github.com> Date: Fri, 3 Sep 2021 15:52:39 -0700 Subject: [PATCH 70/79] Add test verifying PUBSUB NUMPAT behavior (#9209) (cherry picked from commit 8b8f05c86c1f1f002caa1f4e1877020389f167e4) --- tests/unit/pubsub.tcl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl index 1906805a7..03c5dfccb 100644 --- a/tests/unit/pubsub.tcl +++ b/tests/unit/pubsub.tcl @@ -152,6 +152,24 @@ start_server {tags {"pubsub network"}} { r pubsub numsub abc def } {abc 0 def 0} + test "NUMPATs returns the number of unique patterns" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + # Three unique patterns and one that overlaps + psubscribe $rd1 "foo*" + psubscribe $rd2 "foo*" + psubscribe $rd1 "bar*" + psubscribe $rd2 "baz*" + + set patterns [r pubsub numpat] + + # clean up clients + punsubscribe $rd1 + punsubscribe $rd2 + assert_equal 3 $patterns + } + test "Mix SUBSCRIBE and PSUBSCRIBE" { set rd1 [redis_deferring_client] assert_equal {1} [subscribe $rd1 {foo.bar}] From e5e3cd469ccafbb324076832e7ce606748a794f2 Mon Sep 17 00:00:00 2001 From: "zhaozhao.zz" <276441700@qq.com> Date: Wed, 8 Sep 2021 16:07:25 +0800 Subject: [PATCH 71/79] Fix wrong offset when replica pause (#9448) When a replica paused, it would not apply any commands event the command comes from master, if we feed the non-applied command to replication stream, the replication offset would be wrong, and data would be lost after failover(since replica's `master_repl_offset` grows but command is not applied). To fix it, here are the changes: * Don't update replica's replication offset or propagate commands to sub-replicas when it's paused in `commandProcessed`. * Show `slave_read_repl_offset` in info reply. * Add an assert to make sure master client should never be blocked unless pause or module (some modules may use block way to do background (parallel) processing and forward original block module command to the replica, it's not a good way but it can work, so the assert excludes module now, but someday in future all modules should rewrite block command to propagate like what `BLPOP` does). (cherry picked from commit 1b83353dc382959e218191f64d94edb9703552e3) --- src/blocked.c | 5 +++++ src/networking.c | 18 ++++++++++------ src/server.c | 10 +++++++-- tests/unit/pause.tcl | 51 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 9 deletions(-) diff --git a/src/blocked.c b/src/blocked.c index eb110bd35..df3b6a926 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -87,6 +87,11 @@ typedef struct bkinfo { * flag is set client query buffer is not longer processed, but accumulated, * and will be processed when the client is unblocked. */ void blockClient(client *c, int btype) { + /* Master client should never be blocked unless pause or module */ + serverAssert(!(c->flags & CLIENT_MASTER && + btype != BLOCKED_MODULE && + btype != BLOCKED_PAUSE)); + c->flags |= CLIENT_BLOCKED; c->btype = btype; server.blocked_clients++; diff --git a/src/networking.c b/src/networking.c index dcb0a9f24..8a3de6e79 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1991,19 +1991,23 @@ int processMultibulkBuffer(client *c) { * 2. In the case of master clients, the replication offset is updated. * 3. Propagate commands we got from our master to replicas down the line. */ void commandProcessed(client *c) { + /* If client is blocked(including paused), just return avoid reset and replicate. + * + * 1. Don't reset the client structure for blocked clients, so that the reply + * callback will still be able to access the client argv and argc fields. + * The client will be reset in unblockClient(). + * 2. Don't update replication offset or propagate commands to replicas, + * since we have not applied the command. */ + if (c->flags & CLIENT_BLOCKED) return; + + resetClient(c); + long long prev_offset = c->reploff; if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) { /* Update the applied replication offset of our master. */ c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos; } - /* Don't reset the client structure for blocked clients, so that the reply - * callback will still be able to access the client argv and argc fields. - * The client will be reset in unblockClient(). */ - if (!(c->flags & CLIENT_BLOCKED)) { - resetClient(c); - } - /* If the client is a master we need to compute the difference * between the applied offset before and after processing the buffer, * to understand how much of the replication stream was actually diff --git a/src/server.c b/src/server.c index 065028e27..8df92fd0d 100644 --- a/src/server.c +++ b/src/server.c @@ -5083,11 +5083,15 @@ sds genRedisInfoString(const char *section) { server.masterhost == NULL ? "master" : "slave"); if (server.masterhost) { long long slave_repl_offset = 1; + long long slave_read_repl_offset = 1; - if (server.master) + if (server.master) { slave_repl_offset = server.master->reploff; - else if (server.cached_master) + slave_read_repl_offset = server.master->read_reploff; + } else if (server.cached_master) { slave_repl_offset = server.cached_master->reploff; + slave_read_repl_offset = server.cached_master->read_reploff; + } info = sdscatprintf(info, "master_host:%s\r\n" @@ -5095,6 +5099,7 @@ sds genRedisInfoString(const char *section) { "master_link_status:%s\r\n" "master_last_io_seconds_ago:%d\r\n" "master_sync_in_progress:%d\r\n" + "slave_read_repl_offset:%lld\r\n" "slave_repl_offset:%lld\r\n" ,server.masterhost, server.masterport, @@ -5103,6 +5108,7 @@ sds genRedisInfoString(const char *section) { server.master ? ((int)(server.unixtime-server.master->lastinteraction)) : -1, server.repl_state == REPL_STATE_TRANSFER, + slave_read_repl_offset, slave_repl_offset ); diff --git a/tests/unit/pause.tcl b/tests/unit/pause.tcl index 67b684d36..21b9680df 100644 --- a/tests/unit/pause.tcl +++ b/tests/unit/pause.tcl @@ -195,6 +195,57 @@ start_server {tags {"pause network"}} { $rd close } + start_server {tags {needs:repl external:skip}} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + + # Avoid PINGs + $master config set repl-ping-replica-period 3600 + r replicaof $master_host $master_port + + wait_for_condition 50 100 { + [s master_link_status] eq {up} + } else { + fail "Replication not started." + } + + test "Test when replica paused, offset would not grow" { + $master set foo bar + set old_master_offset [status $master master_repl_offset] + + wait_for_condition 50 100 { + [s slave_repl_offset] == [status $master master_repl_offset] + } else { + fail "Replication offset not matched." + } + + r client pause 100000 write + $master set foo2 bar2 + + # Make sure replica received data from master + wait_for_condition 50 100 { + [s slave_read_repl_offset] == [status $master master_repl_offset] + } else { + fail "Replication not work." + } + + # Replica would not apply the write command + assert {[s slave_repl_offset] == $old_master_offset} + r get foo2 + } {} + + test "Test replica offset would grow after unpause" { + r client unpause + wait_for_condition 50 100 { + [s slave_repl_offset] == [status $master master_repl_offset] + } else { + fail "Replication not continue." + } + r get foo2 + } {bar2} + } + # Make sure we unpause at the end r client unpause } From 1e21863e20441aa924635e85a2e91c888007f8c2 Mon Sep 17 00:00:00 2001 From: yvette903 <49490087+yvette903@users.noreply.github.com> Date: Thu, 9 Sep 2021 18:44:48 +0800 Subject: [PATCH 72/79] Fix: client pause uses an old timeout (#9477) A write request may be paused unexpectedly because `server.client_pause_end_time` is old. **Recreate this:** redis-cli -p 6379 127.0.0.1:6379> client pause 500000000 write OK 127.0.0.1:6379> client unpause OK 127.0.0.1:6379> client pause 10000 write OK 127.0.0.1:6379> set key value The write request `set key value` is paused util the timeout of 500000000 milliseconds was reached. **Fix:** reset `server.client_pause_end_time` = 0 in `unpauseClients` (cherry picked from commit f560531d5b8a6e6d810b62114e69a5ffda7730f7) --- src/networking.c | 1 + src/server.c | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/networking.c b/src/networking.c index 8a3de6e79..bf4a8daa0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -3364,6 +3364,7 @@ void unpauseClients(void) { client *c; server.client_pause_type = CLIENT_PAUSE_OFF; + server.client_pause_end_time = 0; /* Unblock all of the clients so they are reprocessed. */ listRewind(server.paused_clients,&li); diff --git a/src/server.c b/src/server.c index 8df92fd0d..58c0a3d6a 100644 --- a/src/server.c +++ b/src/server.c @@ -2764,6 +2764,10 @@ void initServerConfig(void) { * Redis 5. However it is possible to revert it via redis.conf. */ server.lua_always_replicate_commands = 1; + /* Client Pause related */ + server.client_pause_type = CLIENT_PAUSE_OFF; + server.client_pause_end_time = 0; + initConfigValues(); } From fe6cfa9615270bc0f8c57ca2d87302ae394d2541 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Sat, 11 Sep 2021 20:54:09 +0100 Subject: [PATCH 73/79] TLS build fix on OpenBSD when built with LibreSSL. (#9486) (cherry picked from commit 418c2e79313b367e64e47d38edd59f9f22a3b4fa) --- src/tls.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tls.c b/src/tls.c index 0f76256d2..c36cc98fe 100644 --- a/src/tls.c +++ b/src/tls.c @@ -176,7 +176,8 @@ void tlsCleanup(void) { redis_tls_client_ctx = NULL; } - #if OPENSSL_VERSION_NUMBER >= 0x10100000L + #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) + // unavailable on LibreSSL OPENSSL_cleanup(); #endif } From dd65d556348e1fb45e7996f236c79885888d0b72 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Sun, 26 Sep 2021 18:46:22 +0300 Subject: [PATCH 74/79] Fix stream sanitization for non-int first value (#9553) This was recently broken in #9321 when we validated stream IDs to be integers but did that after to the stepping next record instead of before. (cherry picked from commit 5a4ab7c7d2da1773c5ed3dcfc6e367b5af03a33e) --- src/t_stream.c | 2 +- tests/integration/rdb.tcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_stream.c b/src/t_stream.c index 7e67c1b0e..51ac2a31d 100644 --- a/src/t_stream.c +++ b/src/t_stream.c @@ -3628,12 +3628,12 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) { p = next; if (!lpValidateNext(lp, &next, size)) return 0; /* entry id */ - p = next; if (!lpValidateNext(lp, &next, size)) return 0; lpGetIntegerIfValid(p, &valid_record); if (!valid_record) return 0; p = next; if (!lpValidateNext(lp, &next, size)) return 0; lpGetIntegerIfValid(p, &valid_record); if (!valid_record) return 0; + p = next; if (!lpValidateNext(lp, &next, size)) return 0; if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) { /* num-of-fields */ diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 9e1c2651a..e652e5573 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -45,7 +45,7 @@ start_server [list overrides [list "dir" $server_path] keep_persistence true] { test {Test RDB stream encoding} { for {set j 0} {$j < 1000} {incr j} { if {rand() < 0.9} { - r xadd stream * foo $j + r xadd stream * foo abc } else { r xadd stream * bar $j } From dc91aca7b0a793a4d3ea17f9ae31c541ca83099b Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 8 Aug 2021 18:30:17 +0300 Subject: [PATCH 75/79] Propagate OPENSSL_PREFIX to hiredis. (#9345) --- src/Makefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index cf3e8c031..830e80a44 100644 --- a/src/Makefile +++ b/src/Makefile @@ -121,13 +121,15 @@ ifeq ($(uname_S),Darwin) # must be referenced explicitly during build. ifeq ($(uname_M),arm64) # Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX - OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include - OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib + OPENSSL_PREFIX=/opt/homebrew/opt/openssl else # Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX - OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include - OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib + OPENSSL_PREFIX=/usr/local/opt/openssl endif +OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include +OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib +# Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles +export OPENSSL_PREFIX else ifeq ($(uname_S),AIX) # AIX From 0708720d3c03aa5df6f2426a6467aa2985b3b0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yunier=20P=C3=A9rez?= Date: Thu, 30 Sep 2021 07:51:19 -0500 Subject: [PATCH 76/79] Allow to override OPENSSL_PREFIX (#9567) While the original issue was on Linux, this should work for other platforms as well. --- deps/hiredis/Makefile | 7 ++++++- src/Makefile | 16 ++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/deps/hiredis/Makefile b/deps/hiredis/Makefile index a8d37a2eb..7e41c97a5 100644 --- a/deps/hiredis/Makefile +++ b/deps/hiredis/Makefile @@ -77,7 +77,12 @@ ifeq ($(USE_SSL),1) endif ifeq ($(uname_S),Linux) - SSL_LDFLAGS=-lssl -lcrypto + ifdef OPENSSL_PREFIX + CFLAGS+=-I$(OPENSSL_PREFIX)/include + SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto + else + SSL_LDFLAGS=-lssl -lcrypto + endif else OPENSSL_PREFIX?=/usr/local/opt/openssl CFLAGS+=-I$(OPENSSL_PREFIX)/include diff --git a/src/Makefile b/src/Makefile index 830e80a44..1001f5b39 100644 --- a/src/Makefile +++ b/src/Makefile @@ -121,15 +121,11 @@ ifeq ($(uname_S),Darwin) # must be referenced explicitly during build. ifeq ($(uname_M),arm64) # Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX - OPENSSL_PREFIX=/opt/homebrew/opt/openssl + OPENSSL_PREFIX?=/opt/homebrew/opt/openssl else # Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX - OPENSSL_PREFIX=/usr/local/opt/openssl + OPENSSL_PREFIX?=/usr/local/opt/openssl endif -OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include -OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib -# Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles -export OPENSSL_PREFIX else ifeq ($(uname_S),AIX) # AIX @@ -190,6 +186,14 @@ endif endif endif endif + +ifdef OPENSSL_PREFIX + OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include + OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib + # Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles + export OPENSSL_PREFIX +endif + # Include paths to dependencies FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram From e8874ee3878f5de325bd46b2d9b75d1f5a564e84 Mon Sep 17 00:00:00 2001 From: sundb Date: Thu, 29 Jul 2021 16:53:21 +0800 Subject: [PATCH 77/79] Fix missing check for sanitize_dump in corrupt-dump-fuzzer test (#9285) this means the assertion that checks that when deep sanitization is enabled, there are no crashes, was missing. (cherry picked from commit 3db0f1a284e4fba703419b892b2d5b8d385afc06) --- tests/integration/corrupt-dump-fuzzer.tcl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl index 4fb503b8e..1c3da9531 100644 --- a/tests/integration/corrupt-dump-fuzzer.tcl +++ b/tests/integration/corrupt-dump-fuzzer.tcl @@ -117,7 +117,7 @@ foreach sanitize_dump {no yes} { set report_and_restart true incr stat_terminated_in_restore write_log_line 0 "corrupt payload: $printable_dump" - if {$sanitize_dump == 1} { + if {$sanitize_dump == yes} { puts "Server crashed in RESTORE with payload: $printable_dump" } } @@ -140,7 +140,7 @@ foreach sanitize_dump {no yes} { set by_signal [count_log_message 0 "crashed by signal"] incr stat_terminated_by_signal $by_signal - if {$by_signal != 0 || $sanitize_dump == 1 } { + if {$by_signal != 0 || $sanitize_dump == yes} { puts "Server crashed (by signal: $by_signal), with payload: $printable_dump" set print_commands true } @@ -186,7 +186,7 @@ foreach sanitize_dump {no yes} { } } # if we run sanitization we never expect the server to crash at runtime - if { $sanitize_dump == 1} { + if {$sanitize_dump == yes} { assert_equal $stat_terminated_in_restore 0 assert_equal $stat_terminated_in_traffic 0 } From c7f304e118659eee749e780498de32be509c7491 Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 5 Aug 2021 22:57:05 +0300 Subject: [PATCH 78/79] corrupt-dump-fuzzer test, avoid creating junk keys (#9302) The execution of the RPOPLPUSH command by the fuzzer created junk keys, that were later being selected by RANDOMKEY and modified. This also meant that lists were statistically tested more than other files. Fix the fuzzer not to pass junk key names to RPOPLPUSH, and add a check that detects that new keys are not added by the fuzzer to detect future similar issues. (cherry picked from commit 3f3f678a4741e6af18230ee1862d9ced7af79faf) --- tests/integration/corrupt-dump-fuzzer.tcl | 7 +++++++ tests/support/util.tcl | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/integration/corrupt-dump-fuzzer.tcl b/tests/integration/corrupt-dump-fuzzer.tcl index 1c3da9531..86cd8121b 100644 --- a/tests/integration/corrupt-dump-fuzzer.tcl +++ b/tests/integration/corrupt-dump-fuzzer.tcl @@ -90,6 +90,7 @@ foreach sanitize_dump {no yes} { r debug set-skip-checksum-validation 1 set start_time [clock seconds] generate_types + set dbsize [r dbsize] r save set cycle 0 set stat_terminated_in_restore 0 @@ -133,6 +134,12 @@ foreach sanitize_dump {no yes} { set sent [generate_fuzzy_traffic_on_key "_$k" 1] ;# traffic for 1 second incr stat_traffic_commands_sent [llength $sent] r del "_$k" ;# in case the server terminated, here's where we'll detect it. + if {$dbsize != [r dbsize]} { + puts "unexpected keys" + puts "keys: [r keys *]" + puts $sent + exit 1 + } } err ] } { # if the server terminated update stats and restart it set report_and_restart true diff --git a/tests/support/util.tcl b/tests/support/util.tcl index d6717f6e1..1d098b543 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -608,6 +608,7 @@ proc generate_fuzzy_traffic_on_key {key duration} { set arity [lindex $cmd_info 1] set arity [expr $arity < 0 ? - $arity: $arity] set firstkey [lindex $cmd_info 3] + set lastkey [lindex $cmd_info 4] set i 1 if {$cmd == "XINFO"} { lappend cmd "STREAM" @@ -637,7 +638,7 @@ proc generate_fuzzy_traffic_on_key {key duration} { incr i 4 } for {} {$i < $arity} {incr i} { - if {$i == $firstkey} { + if {$i == $firstkey || $i == $lastkey} { lappend cmd $key } else { lappend cmd [randomValue] From 46dddd680cdae664e1af028af165e76635f8deee Mon Sep 17 00:00:00 2001 From: Oran Agra Date: Thu, 30 Sep 2021 23:44:16 +0300 Subject: [PATCH 79/79] Redis 6.2.6 --- 00-RELEASENOTES | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ src/version.h | 4 ++-- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index a5fb897ee..62d1def15 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -11,6 +11,60 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 6.2.6 Released Mon Oct 4 12:00:00 IDT 2021 +================================================================================ + +Upgrade urgency: SECURITY, contains fixes to security issues. + +Security Fixes: +* (CVE-2021-41099) Integer to heap buffer overflow handling certain string + commands and network payloads, when proto-max-bulk-len is manually configured + to a non-default, very large value [reported by yiyuaner]. +* (CVE-2021-32762) Integer to heap buffer overflow issue in redis-cli and + redis-sentinel parsing large multi-bulk replies on some older and less common + platforms [reported by Microsoft Vulnerability Research]. +* (CVE-2021-32687) Integer to heap buffer overflow with intsets, when + set-max-intset-entries is manually configured to a non-default, very large + value [reported by Pawel Wieczorkiewicz, AWS]. +* (CVE-2021-32675) Denial Of Service when processing RESP request payloads with + a large number of elements on many connections. +* (CVE-2021-32672) Random heap reading issue with Lua Debugger [reported by + Meir Shpilraien]. +* (CVE-2021-32628) Integer to heap buffer overflow handling ziplist-encoded + data types, when configuring a large, non-default value for + hash-max-ziplist-entries, hash-max-ziplist-value, zset-max-ziplist-entries + or zset-max-ziplist-value [reported by sundb]. +* (CVE-2021-32627) Integer to heap buffer overflow issue with streams, when + configuring a non-default, large value for proto-max-bulk-len and + client-query-buffer-limit [reported by sundb]. +* (CVE-2021-32626) Specially crafted Lua scripts may result with Heap buffer + overflow [reported by Meir Shpilraien]. + +Bug fixes that involve behavior changes: +* GEO* STORE with empty source key deletes the destination key and return 0 (#9271) + Previously it would have returned an empty array like the non-STORE variant. +* PUBSUB NUMPAT replies with number of patterns rather than number of subscriptions (#9209) + This actually changed in 6.2.0 but was overlooked and omitted from the release notes. + +Bug fixes that are only applicable to previous releases of Redis 6.2: +* Fix CLIENT PAUSE, used an old timeout from previous PAUSE (#9477) +* Fix CLIENT PAUSE in a replica would mess the replication offset (#9448) +* Add some missing error statistics in INFO errorstats (#9328) + +Other bug fixes: +* Fix incorrect reply of COMMAND command key positions for MIGRATE command (#9455) +* Fix appendfsync to always guarantee fsync before reply, on MacOS and FreeBSD (kqueue) (#9416) +* Fix the wrong mis-detection of sync_file_range system call, affecting performance (#9371) + +CLI tools: +* When redis-cli received ASK response, it didn't handle it (#8930) + +Improvements: +* Add latency monitor sample when key is deleted via lazy expire (#9317) +* Sanitize corrupt payload improvements (#9321, #9399) +* Delete empty keys when loading RDB file or handling a RESTORE command (#9297, #9349) + ================================================================================ Redis 6.2.5 Released Wed Jul 21 16:32:19 IDT 2021 ================================================================================ diff --git a/src/version.h b/src/version.h index cd2ff3a6e..e07d557e3 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "6.2.5" -#define REDIS_VERSION_NUM 0x00060205 +#define REDIS_VERSION "6.2.6" +#define REDIS_VERSION_NUM 0x00060206